From 0d496559513cc848e7f566d3a3117e55e948ff3c Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 26 Feb 2019 18:08:57 +0300 Subject: [PATCH] Rollup support for TSVB (#28762) * Added a feature of rollup search on the UI side Signed-off-by: Alexey Antonov * Rollup Feature - initial commit * Revert "Added a feature of rollup search on the UI side" This reverts commit 9568b0970b16f5102f50b748bb4d691a8612c2c2. # Conflicts: # src/legacy/core_plugins/metrics/public/components/index_pattern.js * Remove the 'label' property from the search strategies * Changed search by strategy from the last * add single search request * rollup_search_strategy add base implementation of isViable method * rollup_search_strategy add base implementation of isViable method -fix * Changed requests due to search request type * refactoring of import Base classes / remove '../../../../../../ * remove extra await * rollup_search_strategy. Refactoring of isRollupJobExists method * remove question * Add support of annotations and table data * skeleton for adding Search Strategy restrictions * Add rollup search capabilities * apply search strategy for annotations request * set fields capabilities for rollup strategy * add timezone, interval into SearchCapabilities * Add fields from capabilities * add timezone, interval into SearchCapabilities * fix default timezone * Merging of two Rollup Jobs was removed * move getFieldsForWildcard to searchStrategy * Fix TSVB search requests should have a timeout # Conflicts: # src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js * Add unit test * apply getEsShardTimeout for annorations/get_request_params, series/get_request_params * rename metrics -> tsvb * search_strategies_register refactoring: move 'add' method from class * Add merge rollup capabilities with fields * Add merge rollup capabilities with fields - small fixes * Add support of 'Everything' aggregation for Rollup Search * Return back metrics plugin * remove 'metrics' from the X-pack\rollup require * Fix test cases * fix broken test: fail: "apis InfraOps GraphQL Endpoints metrics should basically work" * rollup search - split by terms is not working * Add count metric * /get_bucket_size.js. Add support of 'auto' interval, Add support of gte intervals e.g.: >=1m * fix build_request_body test * [Rollup] [Phase 1] Error handling - rollup search errors should be more user friendly * [Rollup] [Phase 1] Table View - research the query to ES - sorting is not wokring * Merge #26006 into rollup # Conflicts: # src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js # src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js # src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js # src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js * Add table view support * fix broken build * fix broken build * [Rollup] [Phase 1] - write new tests (rollup_search_request, rollup_search_strategy) * [Rollup] [Phase 1] - write tests for rollup_search_capabilities * Add test on default_search_capabilities, abstract_search_strategy, search_strategies_register * Add test cases for search_requests folder * [Rollup] [Phase 1] - write tests for rollup_search_strategy * FIx broken build * remove todo * fix calculation of interval value for rollup search * add unit tests * Remove default exports * fix PR comments * fix calendar intervals --- src/legacy/core_plugins/metrics/index.js | 10 +- .../metrics/public/components/_error.scss | 31 ++-- .../metrics/public/components/error.js | 8 +- .../public/components/visualization.js | 4 +- .../metrics/server/lib/get_fields.js | 13 +- .../default_search_capabilities.js | 58 +++++++ .../default_search_capabilities.test.js | 106 ++++++++++++ .../index.js} | 8 +- .../search_strategies_register.js | 55 +++++++ .../search_strategies_register.test.js | 94 +++++++++++ .../searh_requests/abstract_request.js} | 16 +- .../searh_requests/abstract_request.test.js | 49 ++++++ .../searh_requests/multi_search_request.js | 34 ++++ .../multi_search_request.test.js | 61 +++++++ .../searh_requests/search_request.js | 37 +++++ .../searh_requests/search_request.test.js | 78 +++++++++ .../searh_requests/single_search_request.js | 34 ++++ .../single_search_request.test.js | 61 +++++++ .../strategies/abstract_search_strategy.js | 42 +++++ .../abstract_search_strategy.test.js | 83 ++++++++++ .../strategies/default_search_strategy.js | 43 +++++ .../default_search_strategy.test.js | 69 ++++++++ .../__tests__/helpers/get_agg_value.js | 9 ++ .../__tests__/helpers/get_es_shard_timeout.js | 2 +- .../annorations/build_request_body.js | 39 +++++ .../annorations/get_request_params.js | 44 +++++ .../server/lib/vis_data/get_annotations.js | 70 +++----- .../server/lib/vis_data/get_series_data.js | 68 ++++---- .../server/lib/vis_data/get_table_data.js | 24 ++- .../lib/vis_data/helpers/get_agg_value.js | 66 ++++---- .../lib/vis_data/helpers/get_bucket_size.js | 52 ++++-- .../vis_data/helpers/get_es_shard_timeout.js | 2 +- .../server/lib/vis_data/helpers/get_splits.js | 27 ++-- .../server/lib/vis_data/helpers/index.js | 2 - .../lib/vis_data/helpers/metric_types.js | 31 ++++ .../lib/vis_data/helpers/unit_to_seconds.js | 29 +++- .../annotations/date_histogram.js | 7 +- .../request_processors/annotations/query.js | 4 +- .../series/__tests__/date_histogram.js | 28 +++- .../series/date_histogram.js | 9 +- .../request_processors/series/index.js | 4 +- .../series/metric_buckets.js | 4 +- .../series/normalize_query.js | 47 ++++++ .../series/sibling_buckets.js | 4 +- .../request_processors/table/filter_ratios.js | 4 +- .../request_processors/table/index.js | 4 +- .../table/normalize_query.js | 51 ++++++ .../response_processors/series/math.js | 4 +- .../response_processors/series/percentile.js | 4 +- .../series/std_deviation_bands.js | 4 +- .../series/std_deviation_sibling.js | 4 +- .../response_processors/series/std_metric.js | 4 +- .../response_processors/series/std_sibling.js | 4 +- .../series/__tests__/build_request_body.js | 70 ++++---- .../lib/vis_data/series/build_request_body.js | 19 ++- .../lib/vis_data/series/get_request_params.js | 28 ++-- .../vis_data/series/handle_response_body.js | 8 +- .../lib/vis_data/table/build_request_body.js | 4 +- .../lib/vis_data/table/process_bucket.js | 13 +- .../__snapshots__/build_pipeline.test.js.snap | 2 +- .../loader/pipeline_helpers/build_pipeline.ts | 12 +- x-pack/plugins/rollup/index.js | 3 + .../lib/merge_capabilities_with_fields.js | 67 ++++++++ .../server/lib/search_strategies/index.js | 7 + .../register_rollup_search_strategy.js | 27 ++++ .../register_rollup_search_strategy.test.js | 52 ++++++ .../rollup_search_capabilities.js | 53 ++++++ .../rollup_search_capabilities.test.js | 147 +++++++++++++++++ .../rollup_search_request.js | 21 +++ .../rollup_search_request.test.js | 51 ++++++ .../rollup_search_strategy.js | 67 ++++++++ .../rollup_search_strategy.test.js | 153 ++++++++++++++++++ .../server/routes/api/index_patterns.js | 58 +------ 73 files changed, 2149 insertions(+), 362 deletions(-) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js rename src/legacy/core_plugins/metrics/server/lib/{vis_data/helpers/extended_stats_types.js => search_strategies/index.js} (91%) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js rename src/legacy/core_plugins/metrics/server/lib/{vis_data/build_annotation_request.js => search_strategies/searh_requests/abstract_request.js} (67%) create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js create mode 100644 src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js create mode 100644 x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/index.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js create mode 100644 x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js diff --git a/src/legacy/core_plugins/metrics/index.js b/src/legacy/core_plugins/metrics/index.js index 1258cedf8c7f1..b0b3dbb8377e0 100644 --- a/src/legacy/core_plugins/metrics/index.js +++ b/src/legacy/core_plugins/metrics/index.js @@ -21,6 +21,7 @@ import { resolve } from 'path'; import fieldsRoutes from './server/routes/fields'; import visDataRoutes from './server/routes/vis'; +import { SearchStrategiesRegister } from './server/lib/search_strategies/search_strategies_register'; export default function (kibana) { return new kibana.Plugin({ @@ -28,7 +29,7 @@ export default function (kibana) { uiExports: { visTypes: [ - 'plugins/metrics/kbn_vis_types' + 'plugins/metrics/kbn_vis_types', ], styleSheetPaths: resolve(__dirname, 'public/index.scss'), }, @@ -37,16 +38,15 @@ export default function (kibana) { return Joi.object({ enabled: Joi.boolean().default(true), chartResolution: Joi.number().default(150), - minimumBucketSize: Joi.number().default(10) + minimumBucketSize: Joi.number().default(10), }).default(); }, - init(server) { fieldsRoutes(server); visDataRoutes(server); - } - + SearchStrategiesRegister.init(server); + }, }); } diff --git a/src/legacy/core_plugins/metrics/public/components/_error.scss b/src/legacy/core_plugins/metrics/public/components/_error.scss index 0718bf58b0ce2..efe378bb791c2 100644 --- a/src/legacy/core_plugins/metrics/public/components/_error.scss +++ b/src/legacy/core_plugins/metrics/public/components/_error.scss @@ -1,15 +1,16 @@ -.tvbError__additional, -.tvbError__stack { - margin-top: $euiSizeS; -} - -// EUITODO: Convert to EuiCodeBlock -.tvbError__stack { - padding: $euiSizeS; - background: $euiCodeBlockBackgroundColor; - color: $euiCodeBlockColor; - line-height: $euiLineHeight; - font-family: $euiCodeFontFamily; - font-weight: $euiFontWeightRegular; - white-space: pre-wrap; -} +.tvbError__title, +.tvbError__additional, +.tvbError__stack { + margin-top: $euiSizeS; +} + +// EUITODO: Convert to EuiCodeBlock +.tvbError__stack { + padding: $euiSizeS; + background: $euiCodeBlockBackgroundColor; + color: $euiCodeBlockColor; + line-height: $euiLineHeight; + font-family: $euiCodeFontFamily; + font-weight: $euiFontWeightRegular; + white-space: pre-wrap; +} diff --git a/src/legacy/core_plugins/metrics/public/components/error.js b/src/legacy/core_plugins/metrics/public/components/error.js index e06d2eaa3ea82..bc3272605e579 100644 --- a/src/legacy/core_plugins/metrics/public/components/error.js +++ b/src/legacy/core_plugins/metrics/public/components/error.js @@ -23,10 +23,12 @@ import React from 'react'; import _ from 'lodash'; import { FormattedMessage } from '@kbn/i18n/react'; +const guidPattern = /\[[[a-f\d-\\]{36}\]/g; + function ErrorComponent(props) { const { error } = props; let additionalInfo; - const type = _.get(error, 'error.caused_by.type'); + const type = _.get(error, 'error.caused_by.type') || _.get(error, 'error.type'); let reason = _.get(error, 'error.caused_by.reason'); const title = _.get(error, 'error.caused_by.title'); @@ -34,6 +36,10 @@ function ErrorComponent(props) { reason = _.get(error, 'message'); } + if (['runtime_exception', 'illegal_argument_exception'].includes(type)) { + reason = _.get(error, 'error.reason').replace(guidPattern, ``); + } + if (type === 'script_exception') { const scriptStack = _.get(error, 'error.caused_by.script_stack'); reason = _.get(error, 'error.caused_by.caused_by.reason'); diff --git a/src/legacy/core_plugins/metrics/public/components/visualization.js b/src/legacy/core_plugins/metrics/public/components/visualization.js index 1af0f8eb85a9e..6e1fcea29016d 100644 --- a/src/legacy/core_plugins/metrics/public/components/visualization.js +++ b/src/legacy/core_plugins/metrics/public/components/visualization.js @@ -27,7 +27,7 @@ import topN from './vis_types/top_n/vis'; import table from './vis_types/table/vis'; import gauge from './vis_types/gauge/vis'; import markdown from './vis_types/markdown/vis'; -import Error from './error'; +import ErrorComponent from './error'; import NoData from './no_data'; const types = { @@ -46,7 +46,7 @@ function Visualization(props) { if (error) { return (
- +
); } diff --git a/src/legacy/core_plugins/metrics/server/lib/get_fields.js b/src/legacy/core_plugins/metrics/server/lib/get_fields.js index c9588d057614d..d9830bca9b850 100644 --- a/src/legacy/core_plugins/metrics/server/lib/get_fields.js +++ b/src/legacy/core_plugins/metrics/server/lib/get_fields.js @@ -16,13 +16,16 @@ * specific language governing permissions and limitations * under the License. */ - +import { SearchStrategiesRegister } from './search_strategies/search_strategies_register'; import { uniq } from 'lodash'; export async function getFields(req) { - const { indexPatternsService } = req.pre; - const index = req.query.index || '*'; - const resp = await indexPatternsService.getFieldsForWildcard({ pattern: index }); - const fields = resp.filter(field => field.aggregatable); + const indexPattern = req.query.index || '*'; + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + + const fields = (await searchStrategy + .getFieldsForWildcard(req, indexPattern, capabilities)) + .filter(field => field.aggregatable); + return uniq(fields, field => field.name); } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js new file mode 100644 index 0000000000000..33e268577bf04 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.js @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { convertIntervalToUnit, parseInterval } from '../vis_data/helpers/unit_to_seconds'; + +const getTimezoneFromRequest = request => { + return request.payload.timerange.timezone; +}; + +export class DefaultSearchCapabilities { + constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { + this.request = request; + this.batchRequestsSupport = batchRequestsSupport; + this.fieldsCapabilities = fieldsCapabilities; + } + + get defaultTimeInterval() { + return null; + } + + get searchTimezone() { + return getTimezoneFromRequest(this.request); + } + + parseInterval(interval) { + return parseInterval(interval); + } + + convertIntervalToUnit(intervalString, unit) { + const parsedInterval = this.parseInterval(intervalString); + + if (parsedInterval.unit !== unit) { + return convertIntervalToUnit(intervalString, unit); + } + + return parsedInterval; + } + + getValidTimeInterval(intervalString) { + // Default search capabilities doesn't have any restrictions for the interval string + return intervalString; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js new file mode 100644 index 0000000000000..5d41e03722d9d --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/default_search_capabilities.test.js @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { DefaultSearchCapabilities } from './default_search_capabilities'; + +describe('DefaultSearchCapabilities', () => { + let defaultSearchCapabilities; + let batchRequestsSupport; + let req; + + beforeEach(() => { + req = {}; + batchRequestsSupport = true; + defaultSearchCapabilities = new DefaultSearchCapabilities(req, batchRequestsSupport); + }); + + test('should init default search capabilities', () => { + expect(defaultSearchCapabilities.request).toBe(req); + expect(defaultSearchCapabilities.batchRequestsSupport).toBe(batchRequestsSupport); + expect(defaultSearchCapabilities.fieldsCapabilities).toEqual({}); + }); + + test('should return defaultTimeInterval', () => { + expect(defaultSearchCapabilities.defaultTimeInterval).toBe(null); + }); + + test('should return Search Timezone', () => { + defaultSearchCapabilities.request = { + payload: { + timerange: { + timezone: 'UTC' + } + } + }; + + expect(defaultSearchCapabilities.searchTimezone).toEqual('UTC'); + }); + + test('should return a valid time interval', () => { + expect(defaultSearchCapabilities.getValidTimeInterval('20m')).toBe('20m'); + }); + + test('should parse interval', () => { + expect(defaultSearchCapabilities.parseInterval('120s')).toEqual({ + value: 120, + unit: 's' + }); + + expect(defaultSearchCapabilities.parseInterval('20m')).toEqual({ + value: 20, + unit: 'm' + }); + + expect(defaultSearchCapabilities.parseInterval('1y')).toEqual({ + value: 1, + unit: 'y' + }); + }); + + test('should convert interval string into different unit', () => { + expect(defaultSearchCapabilities.convertIntervalToUnit('120s', 's')).toEqual({ + value: 120, + unit: 's' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('60m', 'h')).toEqual({ + value: 1, + unit: 'h' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('4w', 'M')).toEqual({ + value: 1, + unit: 'M' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('1y', 'w')).toEqual({ + value: 48, + unit: 'w' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('60s', 'm')).toEqual({ + value: 1, + unit: 'm' + }); + + expect(defaultSearchCapabilities.convertIntervalToUnit('1s', 'ms')).toEqual({ + value: 1000, + unit: 'ms' + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js similarity index 91% rename from src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js rename to src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js index e206a1ce5c23e..512894f30a619 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/extended_stats_types.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/index.js @@ -17,10 +17,4 @@ * under the License. */ -export default [ - 'std_deviation', - 'variance', - 'sum_of_squares' -]; - - +export { SearchStrategiesRegister } from './search_strategies_register'; diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js new file mode 100644 index 0000000000000..159e25191fc94 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.js @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchStrategy } from './strategies/abstract_search_strategy'; +import { AbstractSearchRequest } from './searh_requests/abstract_request'; +import { DefaultSearchStrategy } from './strategies/default_search_strategy'; +import { DefaultSearchCapabilities } from './default_search_capabilities'; + +const strategies = []; + +const addStrategy = searchStrategy => { + if (searchStrategy instanceof AbstractSearchStrategy) { + strategies.unshift(searchStrategy); + } + return strategies; +}; + +export class SearchStrategiesRegister { + static init(server) { + server.expose('AbstractSearchStrategy', AbstractSearchStrategy); + server.expose('AbstractSearchRequest', AbstractSearchRequest); + server.expose('DefaultSearchCapabilities', DefaultSearchCapabilities); + server.expose('addSearchStrategy', searchStrategy => addStrategy(searchStrategy)); + + addStrategy(new DefaultSearchStrategy(server)); + } + + static async getViableStrategy(req, indexPattern) { + for (const searchStrategy of strategies) { + const { isViable, capabilities } = await searchStrategy.checkForViability(req, indexPattern); + + if (isViable) { + return { + searchStrategy, + capabilities, + }; + } + } + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js new file mode 100644 index 0000000000000..abe25b4737b7e --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/search_strategies_register.test.js @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { SearchStrategiesRegister } from './search_strategies_register'; +import { AbstractSearchStrategy } from './strategies/abstract_search_strategy'; +import { DefaultSearchStrategy } from './strategies/default_search_strategy'; +import { AbstractSearchRequest } from './searh_requests/abstract_request'; +import { DefaultSearchCapabilities } from './default_search_capabilities'; + +class MockSearchStrategy extends AbstractSearchStrategy { + checkForViability() { + return { + isViable: true, + capabilities: {} + }; + } +} + +describe('SearchStrategiesRegister', () => { + let server; + let strategies; + let anotherSearchStrategy; + + beforeAll(() => { + server = { + expose: jest.fn((strategy, func) => { + server[strategy] = func; + }) + }; + strategies = [ + ['AbstractSearchStrategy', AbstractSearchStrategy], + ['AbstractSearchRequest', AbstractSearchRequest], + ['DefaultSearchCapabilities', DefaultSearchCapabilities], + ['addSearchStrategy', expect.any(Function)] + ]; + + SearchStrategiesRegister.init(server); + }); + + test('should init strategies register', () => { + expect(server.expose.mock.calls).toEqual(strategies); + expect(server.addSearchStrategy()[0] instanceof DefaultSearchStrategy).toBe(true); + }); + + test('should not add a strategy if it is not an instance of AbstractSearchStrategy', () => { + const addedStrategies = server.addSearchStrategy({}); + + expect(addedStrategies.length).toEqual(1); + expect(addedStrategies[0] instanceof DefaultSearchStrategy).toBe(true); + }); + + test('should return a DefaultSearchStrategy instance', async () => { + const req = {}; + const indexPattern = '*'; + + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + + expect(searchStrategy instanceof DefaultSearchStrategy).toBe(true); + expect(capabilities instanceof DefaultSearchCapabilities).toBe(true); + }); + + test('should add a strategy if it is an instance of AbstractSearchStrategy', () => { + anotherSearchStrategy = new MockSearchStrategy(); + const addedStrategies = server.addSearchStrategy(anotherSearchStrategy); + + expect(addedStrategies.length).toEqual(2); + expect(addedStrategies[0] instanceof AbstractSearchStrategy).toBe(true); + }); + + test('should return a MockSearchStrategy instance', async () => { + const req = {}; + const indexPattern = '*'; + + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, indexPattern); + + expect(searchStrategy instanceof AbstractSearchStrategy).toBe(true); + expect(capabilities).toEqual({}); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js similarity index 67% rename from src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js rename to src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js index 1796f3a0a7927..cfacaf18a5abf 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/build_annotation_request.js +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.js @@ -16,12 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +export class AbstractSearchRequest { + constructor(req, callWithRequest, indexPattern) { + this.req = req; + this.callWithRequest = callWithRequest; + this.indexPattern = indexPattern; + } -import buildProcessorFunction from './build_processor_function'; -import processors from './request_processors/annotations'; - -export default function buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, annotation, esQueryConfig, indexPattern); - const doc = processor({}); - return doc; + search() { + throw new Error('AbstractSearchRequest: search method should be defined'); + } } diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js new file mode 100644 index 0000000000000..16dbd9b580f69 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/abstract_request.test.js @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchRequest } from './abstract_request'; + +describe('AbstractSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + + beforeEach(() => { + req = {}; + callWithRequest = jest.fn(); + indexPattern = 'indexPattern'; + searchRequest = new AbstractSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an AbstractSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should throw an error trying to search', () => { + try { + searchRequest.search(); + } catch (error) { + expect(error instanceof Error).toBe(true); + expect(error.message).toEqual('AbstractSearchRequest: search method should be defined'); + } + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js new file mode 100644 index 0000000000000..793448073397b --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.js @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchRequest } from './abstract_request'; + +const SEARCH_METHOD = 'msearch'; + +export class MultiSearchRequest extends AbstractSearchRequest { + async search(options) { + const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); + const { responses } = await this.callWithRequest(this.req, SEARCH_METHOD, { + ...options, + rest_total_hits_as_int: true, + ignore_throttled: !includeFrozen, + }); + + return responses; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js new file mode 100644 index 0000000000000..48d24f7622796 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/multi_search_request.test.js @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { MultiSearchRequest } from './multi_search_request'; + +describe('MultiSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({ responses: [] }); + indexPattern = 'indexPattern'; + searchRequest = new MultiSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an MultiSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should get the response from elastic msearch', async () => { + const options = {}; + + const responses = await searchRequest.search(options); + + expect(responses).toEqual([]); + expect(req.getUiSettingsService).toHaveBeenCalled(); + expect(getServiceMock).toHaveBeenCalledWith('search:includeFrozen'); + expect(callWithRequest).toHaveBeenCalledWith(req, 'msearch', { + ...options, + rest_total_hits_as_int: true, + ignore_throttled: !includeFrozen, + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js new file mode 100644 index 0000000000000..afab9d37f3018 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.js @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchRequest } from './abstract_request'; + +import { MultiSearchRequest } from './multi_search_request'; +import { SingleSearchRequest } from './single_search_request'; + +export class SearchRequest extends AbstractSearchRequest { + getSearchRequestType(options) { + const isMultiSearch = Array.isArray(options.body); + const SearchRequest = isMultiSearch ? MultiSearchRequest : SingleSearchRequest; + + return new SearchRequest(this.req, this.callWithRequest, this.indexPattern); + } + + async search(options) { + const concreteSearchRequest = this.getSearchRequestType(options); + + return concreteSearchRequest.search(options); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js new file mode 100644 index 0000000000000..608f8abddf95d --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/search_request.test.js @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { SearchRequest } from './search_request'; +import { MultiSearchRequest } from './multi_search_request'; +import { SingleSearchRequest } from './single_search_request'; + +describe('SearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({ responses: [] }); + indexPattern = 'indexPattern'; + searchRequest = new SearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an AbstractSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should return search value', async () => { + const concreteSearchRequest = { + search: jest.fn().mockReturnValue('concreteSearchRequest') + }; + const options = {}; + searchRequest.getSearchRequestType = jest.fn().mockReturnValue(concreteSearchRequest); + + const result = await searchRequest.search(options); + + expect(result).toBe('concreteSearchRequest'); + }); + + test('should return a MultiSearchRequest if options has body as an array', () => { + const options = { + body: [] + }; + + const result = searchRequest.getSearchRequestType(options); + + expect(result instanceof MultiSearchRequest).toBe(true); + }); + + test('should return a SingleSearchRequest if options has body', () => { + const options = {}; + + const result = searchRequest.getSearchRequestType(options); + + expect(result instanceof SingleSearchRequest).toBe(true); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js new file mode 100644 index 0000000000000..df27b890f0fd0 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.js @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchRequest } from './abstract_request'; + +const SEARCH_METHOD = 'search'; + +export class SingleSearchRequest extends AbstractSearchRequest { + async search(options) { + const includeFrozen = await this.req.getUiSettingsService().get('search:includeFrozen'); + const resp = await this.callWithRequest(this.req, SEARCH_METHOD, { + ...options, + index: this.indexPattern, + ignore_throttled: !includeFrozen, + }); + + return [resp]; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js new file mode 100644 index 0000000000000..97cbaa188cee4 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/searh_requests/single_search_request.test.js @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { SingleSearchRequest } from './single_search_request'; + +describe('SingleSearchRequest', () => { + let searchRequest; + let req; + let callWithRequest; + let indexPattern; + let getServiceMock; + let includeFrozen; + + beforeEach(() => { + includeFrozen = false; + getServiceMock = jest.fn().mockResolvedValue(includeFrozen); + req = { + getUiSettingsService: jest.fn().mockReturnValue({ get: getServiceMock }) + }; + callWithRequest = jest.fn().mockReturnValue({}); + indexPattern = 'indexPattern'; + searchRequest = new SingleSearchRequest(req, callWithRequest, indexPattern); + }); + + test('should init an SingleSearchRequest instance', () => { + expect(searchRequest.req).toBe(req); + expect(searchRequest.callWithRequest).toBe(callWithRequest); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.search).toBeDefined(); + }); + + test('should get the response from elastic search', async () => { + const options = {}; + + const responses = await searchRequest.search(options); + + expect(responses).toEqual([{}]); + expect(req.getUiSettingsService).toHaveBeenCalled(); + expect(getServiceMock).toHaveBeenCalledWith('search:includeFrozen'); + expect(callWithRequest).toHaveBeenCalledWith(req, 'search', { + ...options, + index: indexPattern, + ignore_throttled: !includeFrozen, + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js new file mode 100644 index 0000000000000..2df4652e8179c --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.js @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class AbstractSearchStrategy { + constructor(server, callWithRequestFactory, SearchRequest) { + this.getCallWithRequestInstance = req => callWithRequestFactory(server, req); + + this.getSearchRequest = (req, indexPattern) => { + const callWithRequest = this.getCallWithRequestInstance(req); + + return new SearchRequest(req, callWithRequest, indexPattern); + }; + } + + async getFieldsForWildcard(req, indexPattern) { + const { indexPatternsService } = req.pre; + + return await indexPatternsService.getFieldsForWildcard({ + pattern: indexPattern, + }); + } + + checkForViability() { + throw new TypeError('Must override method'); + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js new file mode 100644 index 0000000000000..c063c5047d069 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchStrategy } from './abstract_search_strategy'; + +class SearchRequest { + constructor(req, callWithRequest, indexPattern) { + this.req = req; + this.callWithRequest = callWithRequest; + this.indexPattern = indexPattern; + } +} + +describe('AbstractSearchStrategy', () => { + let abstractSearchStrategy; + let server; + let callWithRequestFactory; + let req; + let mockedFields; + let indexPattern; + + beforeEach(() => { + server = {}; + callWithRequestFactory = jest.fn().mockReturnValue('callWithRequest'); + mockedFields = {}; + indexPattern = '*'; + req = { + pre: { + indexPatternsService: { + getFieldsForWildcard: jest.fn().mockReturnValue(mockedFields) + } + } + }; + + abstractSearchStrategy = new AbstractSearchStrategy(server, callWithRequestFactory, SearchRequest); + }); + + test('should init an AbstractSearchStrategy instance', () => { + expect(abstractSearchStrategy.getCallWithRequestInstance).toBeDefined(); + expect(abstractSearchStrategy.getSearchRequest).toBeDefined(); + expect(abstractSearchStrategy.getFieldsForWildcard).toBeDefined(); + expect(abstractSearchStrategy.checkForViability).toBeDefined(); + }); + + test('should return fields for wildcard', async () => { + const fields = await abstractSearchStrategy.getFieldsForWildcard(req, indexPattern); + + expect(fields).toBe(mockedFields); + expect(req.pre.indexPatternsService.getFieldsForWildcard).toHaveBeenCalledWith({ + pattern: indexPattern, + }); + }); + + test('should invoke callWithRequestFactory with req param passed', () => { + abstractSearchStrategy.getCallWithRequestInstance(req); + + expect(callWithRequestFactory).toHaveBeenCalledWith(server, req); + }); + + test('should return a search request', () => { + const searchRequest = abstractSearchStrategy.getSearchRequest(req, indexPattern); + + expect(searchRequest instanceof SearchRequest).toBe(true); + expect(searchRequest.indexPattern).toBe(indexPattern); + expect(searchRequest.callWithRequest).toBe('callWithRequest'); + expect(searchRequest.req).toBe(req); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js new file mode 100644 index 0000000000000..15a796b5e511a --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.js @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AbstractSearchStrategy } from './abstract_search_strategy'; +import { SearchRequest } from '../searh_requests/search_request'; +import { DefaultSearchCapabilities } from '../default_search_capabilities'; + +const callWithRequestFactory = (server, request) => { + const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data'); + + return callWithRequest; +}; +const batchRequestsSupport = true; + +export class DefaultSearchStrategy extends AbstractSearchStrategy { + name = 'default'; + + constructor(server) { + super(server, callWithRequestFactory, SearchRequest); + } + + checkForViability(req) { + return { + isViable: true, + capabilities: new DefaultSearchCapabilities(req, batchRequestsSupport) + }; + } +} diff --git a/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js new file mode 100644 index 0000000000000..9f1750c1b3db2 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/search_strategies/strategies/default_search_strategy.test.js @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { DefaultSearchStrategy } from './default_search_strategy'; + +describe('DefaultSearchStrategy', () => { + let defaultSearchStrategy; + let server; + let callWithRequest; + let req; + + beforeEach(() => { + server = {}; + callWithRequest = jest.fn(); + req = { + server: { + plugins: { + elasticsearch: { + getCluster: jest.fn().mockReturnValue({ + callWithRequest + }) + } + } + } + }; + defaultSearchStrategy = new DefaultSearchStrategy(server); + }); + + test('should init an DefaultSearchStrategy instance', () => { + expect(defaultSearchStrategy.name).toBe('default'); + expect(defaultSearchStrategy.checkForViability).toBeDefined(); + expect(defaultSearchStrategy.getCallWithRequestInstance).toBeDefined(); + expect(defaultSearchStrategy.getSearchRequest).toBeDefined(); + expect(defaultSearchStrategy.getFieldsForWildcard).toBeDefined(); + }); + + test('should invoke callWithRequestFactory with passed params', () => { + const value = defaultSearchStrategy.getCallWithRequestInstance(req); + + expect(value).toBe(callWithRequest); + expect(req.server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith('data'); + }); + + test('should check a strategy for viability', () => { + const value = defaultSearchStrategy.checkForViability(req); + + expect(value.isViable).toBe(true); + expect(value.capabilities).toEqual({ + request: req, + batchRequestsSupport: true, + fieldsCapabilities: {}, + }); + }); +}); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js index 804d50464b47a..8b6d194c92aad 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_agg_value.js @@ -131,6 +131,15 @@ describe('getAggValue', () => { }, }; + + describe('count', () => { + testAgg( + basicWithDerv, + { id: 'test', type: 'count' }, + 2 + ); + }); + describe('derivative', () => { testAgg( basicWithDerv, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js index aa4a15b820247..4000fa06aa5c5 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/__tests__/helpers/get_es_shard_timeout.js @@ -19,7 +19,7 @@ import sinon from 'sinon'; import { expect } from 'chai'; -import getEsShardTimeout from '../../helpers/get_es_shard_timeout'; +import { getEsShardTimeout } from '../../helpers/get_es_shard_timeout'; describe('getEsShardTimeout', () => { it('should return the elasticsearch.shardTimeout', () => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js new file mode 100644 index 0000000000000..91f55b1eb2200 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/build_request_body.js @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import buildProcessorFunction from '../build_processor_function'; +import processors from '../request_processors/annotations'; + +/** + * Builds annotation request body + * + * @param {...args}: [ + * req: {Object} - a request object, + * panel: {Object} - a panel object, + * annotation: {Object} - an annotation object, + * esQueryConfig: {Object} - es query config object, + * indexPatternObject: {Object} - an index pattern object, + * capabilities: {Object} - a search capabilities object + * ] + * @returns {Object} doc - processed body + */ +export function buildAnnotationRequest(...args) { + const processor = buildProcessorFunction(processors, ...args); + const doc = processor({}); + return doc; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js new file mode 100644 index 0000000000000..47ca00d8fbaf5 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/annorations/get_request_params.js @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { buildAnnotationRequest } from './build_request_body'; +import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; +import { getIndexPatternObject } from '../helpers/get_index_pattern'; + +export async function getAnnotationRequestParams(req, panel, annotation, esQueryConfig, capabilities) { + const bodies = []; + const esShardTimeout = getEsShardTimeout(req); + const indexPattern = annotation.index_pattern; + const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); + const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities); + + if (capabilities.batchRequestsSupport) { + bodies.push({ + index: indexPatternString, + ignoreUnavailable: true, + }); + } + + if (esShardTimeout > 0) { + request.timeout = `${esShardTimeout}ms`; + } + + bodies.push(request); + + return bodies; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js index af37c3f3dab7d..3a71358b92ddb 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_annotations.js @@ -16,11 +16,8 @@ * specific language governing permissions and limitations * under the License. */ - -import buildAnnotationRequest from './build_annotation_request'; import handleAnnotationResponse from './handle_annotation_response'; -import { getIndexPatternObject } from './helpers/get_index_pattern'; -import getEsShardTimeout from './helpers/get_es_shard_timeout'; +import { getAnnotationRequestParams } from './annorations/get_request_params'; function validAnnotation(annotation) { return annotation.index_pattern && @@ -30,55 +27,28 @@ function validAnnotation(annotation) { annotation.template; } -export default async (req, panel, esQueryConfig) => { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const bodiesPromises = panel.annotations - .filter(validAnnotation) - .map(annotation => { - return getAnnotationBody(req, panel, annotation, esQueryConfig); - }); - const bodies = await Promise.all(bodiesPromises); - if (!bodies.length) { - return { - responses: [], - }; - } +export async function getAnnotations(req, panel, esQueryConfig, searchStrategy, capabilities) { + const panelIndexPattern = panel.index_pattern; + const searchRequest = searchStrategy.getSearchRequest(req, panelIndexPattern); + const annotations = panel.annotations.filter(validAnnotation); + + const bodiesPromises = annotations.map(annotation => getAnnotationRequestParams(req, panel, annotation, esQueryConfig, capabilities)); + const body = (await Promise.all(bodiesPromises)) + .reduce((acc, items) => acc.concat(items), []); + + if (!body.length) return { responses: [] }; + try { - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); - const resp = await callWithRequest(req, 'msearch', { - ignore_throttled: !includeFrozen, - rest_total_hits_as_int: true, - body: bodies.reduce((acc, item) => acc.concat(item), []) - }); - const results = {}; - panel.annotations - .filter(validAnnotation) - .forEach((annotation, index) => { - const data = resp.responses[index]; - results[annotation.id] = handleAnnotationResponse(data, annotation); - }); - return results; + const responses = await searchRequest.search({ body }); + + return annotations + .reduce((acc, annotation, index) => { + acc[annotation.id] = handleAnnotationResponse(responses[index], annotation); + + return acc; + }, {}); } catch (error) { if (error.message === 'missing-indices') return { responses: [] }; throw error; } -}; - -async function getAnnotationBody(req, panel, annotation, esQueryConfig) { - const indexPattern = annotation.index_pattern; - const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); - const request = buildAnnotationRequest(req, panel, annotation, esQueryConfig, indexPatternObject); - const esShardTimeout = getEsShardTimeout(req); - - if (esShardTimeout > 0) { - request.timeout = `${esShardTimeout}ms`; - } - - return [ - { - index: indexPatternString, - ignoreUnavailable: true, - }, - request, - ]; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js index 2594db2c5db6d..b6624b8fcc5ea 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_series_data.js @@ -16,50 +16,44 @@ * specific language governing permissions and limitations * under the License. */ - -import getRequestParams from './series/get_request_params'; +import { getSeriesRequestParams } from './series/get_request_params'; import handleResponseBody from './series/handle_response_body'; import handleErrorResponse from './handle_error_response'; -import getAnnotations from './get_annotations'; +import { getAnnotations } from './get_annotations'; +import { SearchStrategiesRegister } from '../search_strategies/search_strategies_register'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; export async function getSeriesData(req, panel) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); + const panelIndexPattern = panel.index_pattern; + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, panelIndexPattern); + const searchRequest = searchStrategy.getSearchRequest(req, panelIndexPattern); const esQueryConfig = await getEsQueryConfig(req); - try { - const bodiesPromises = panel.series.map(series => getRequestParams(req, panel, series, esQueryConfig)); - const bodies = await Promise.all(bodiesPromises); - const params = { - rest_total_hits_as_int: true, - ignore_throttled: !includeFrozen, - body: bodies.reduce((acc, items) => acc.concat(items), []) - }; - return callWithRequest(req, 'msearch', params) - .then(resp => { - const series = resp.responses.map(handleResponseBody(panel)); - return { - [panel.id]: { - id: panel.id, - series: series.reduce((acc, series) => acc.concat(series), []) - } - }; - }) - .then(resp => { - if (!panel.annotations || panel.annotations.length === 0) return resp; - return getAnnotations(req, panel, esQueryConfig).then(annotations => { - resp[panel.id].annotations = annotations; - return resp; - }); - }) - .then(resp => { - resp.type = panel.type; + const bodiesPromises = panel.series.map(series => getSeriesRequestParams(req, panel, series, esQueryConfig, capabilities)); + const body = (await Promise.all(bodiesPromises)) + .reduce((acc, items) => acc.concat(items), []); + + return searchRequest.search({ body }) + .then(data => { + const series = data.map(handleResponseBody(panel)); + return { + [panel.id]: { + id: panel.id, + series: series.reduce((acc, series) => acc.concat(series), []), + }, + }; + }) + .then(resp => { + if (!panel.annotations || panel.annotations.length === 0) return resp; + return getAnnotations(req, panel, esQueryConfig, searchStrategy, capabilities).then(annotations => { + resp[panel.id].annotations = annotations; return resp; - }) - .catch(handleErrorResponse(panel)); - } catch(e) { - return handleErrorResponse(e); - } + }); + }) + .then(resp => { + resp.type = panel.type; + return resp; + }) + .catch(handleErrorResponse(panel)); } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js index 53c18a9c6a1ae..c552022473e00 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/get_table_data.js @@ -16,29 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -import { get } from 'lodash'; + import buildRequestBody from './table/build_request_body'; import handleErrorResponse from './handle_error_response'; +import { get } from 'lodash'; import processBucket from './table/process_bucket'; -import { getIndexPatternObject } from './helpers/get_index_pattern'; +import { SearchStrategiesRegister } from '../search_strategies/search_strategies_register'; import { getEsQueryConfig } from './helpers/get_es_query_uisettings'; - +import { getIndexPatternObject } from './helpers/get_index_pattern'; export async function getTableData(req, panel) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('data'); - const includeFrozen = await req.getUiSettingsService().get('search:includeFrozen'); - + const panelIndexPattern = panel.index_pattern; + const { searchStrategy, capabilities } = await SearchStrategiesRegister.getViableStrategy(req, panelIndexPattern); + const searchRequest = searchStrategy.getSearchRequest(req, panelIndexPattern); const esQueryConfig = await getEsQueryConfig(req); - const indexPattern = panel.index_pattern; - const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); + const { indexPatternObject } = await getIndexPatternObject(req, panelIndexPattern); + const body = buildRequestBody(req, panel, esQueryConfig, indexPatternObject, capabilities); - const params = { - index: indexPatternString, - ignore_throttled: !includeFrozen, - body: buildRequestBody(req, panel, esQueryConfig, indexPatternObject) - }; try { - const resp = await callWithRequest(req, 'search', params); + const [resp] = await searchRequest.search({ body }); const buckets = get(resp, 'aggregations.pivot.buckets', []); return { type: 'table', series: buckets.map(processBucket(panel)) }; } catch (err) { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js index 57a58cf00cfe4..5bf32ee4ff174 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_agg_value.js @@ -18,7 +18,7 @@ */ import { get, includes, max, min, sum } from 'lodash'; -import extendStatsTypes from './extended_stats_types'; +import { EXTENDED_STATS_TYPES, METRIC_TYPES } from './metric_types'; const aggFns = { max, @@ -29,7 +29,7 @@ const aggFns = { export default (row, metric) => { // Extended Stats - if (includes(extendStatsTypes, metric.type)) { + if (includes(EXTENDED_STATS_TYPES, metric.type)) { const isStdDeviation = /^std_deviation/.test(metric.type); const modeIsBounds = ~['upper', 'lower'].indexOf(metric.mode); if (isStdDeviation && modeIsBounds) { @@ -38,38 +38,36 @@ export default (row, metric) => { return get(row, `${metric.id}.${metric.type}`); } - // Percentiles - if (metric.type === 'percentile') { - let percentileKey = `${metric.percent}`; - if (!/\./.test(`${metric.percent}`)) { - percentileKey = `${metric.percent}.0`; - } - return row[metric.id].values[percentileKey]; - } - - if (metric.type === 'percentile_rank') { - const percentileRankKey = `${metric.value}`; - return ( - row[metric.id] && - row[metric.id].values && - row[metric.id].values[percentileRankKey] - ); - } + switch (metric.type) { + case METRIC_TYPES.PERCENTILE: + let percentileKey = `${metric.percent}`; + if (!/\./.test(`${metric.percent}`)) { + percentileKey = `${metric.percent}.0`; + } + return row[metric.id].values[percentileKey]; + case METRIC_TYPES.PERCENTILE_RANK: + const percentileRankKey = `${metric.value}`; + return ( + row[metric.id] && + row[metric.id].values && + row[metric.id].values[percentileRankKey] + ); + case METRIC_TYPES.TOP_HIT: + if (row[metric.id].doc_count === 0) return null; + const hits = get(row, [metric.id, 'docs', 'hits', 'hits'], []); + const values = hits.map(doc => { + return get(doc, `_source.${metric.field}`, 0); + }); + const aggWith = (metric.agg_with && aggFns[metric.agg_with]) || aggFns.avg; + return aggWith(values); + case METRIC_TYPES.COUNT: + return get(row, 'doc_count', null); + default: + // Derivatives + const normalizedValue = get(row, `${metric.id}.normalized_value`, null); - if (metric.type === 'top_hit') { - if (row[metric.id].doc_count === 0) return null; - const hits = get(row, [metric.id, 'docs', 'hits', 'hits'], []); - const values = hits.map(doc => { - return get(doc, `_source.${metric.field}`, 0); - }); - const aggWith = (metric.agg_with && aggFns[metric.agg_with]) || aggFns.avg; - return aggWith(values); + // Everything else + const value = get(row, `${metric.id}.value`, null); + return normalizedValue || value; } - - // Derivatives - const normalizedValue = get(row, `${metric.id}.normalized_value`, null); - - // Everything else - const value = get(row, `${metric.id}.value`, null); - return normalizedValue || value; }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js index cd6cf4206d63c..687cc02853038 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_bucket_size.js @@ -22,33 +22,53 @@ import moment from 'moment'; import unitToSeconds from './unit_to_seconds'; import { INTERVAL_STRING_RE, - GTE_INTERVAL_RE + GTE_INTERVAL_RE, } from '../../../../common/interval_regexp'; -export default (req, interval) => { - const from = moment.utc(req.payload.timerange.min); - const to = moment.utc(req.payload.timerange.max); + +const calculateBucketData = (timeInterval, capabilities) => { + const intervalString = capabilities ? capabilities.getValidTimeInterval(timeInterval) : timeInterval; + const intervalStringMatch = intervalString.match(INTERVAL_STRING_RE); + + let bucketSize = Number(intervalStringMatch[1]) * unitToSeconds(intervalStringMatch[2]); + + // don't go too small + if (bucketSize < 1) { + bucketSize = 1; + } + + return { + bucketSize, + intervalString, + }; +}; + +const getTimeRangeBucketSize = ({ min, max }) => { + const from = moment.utc(min); + const to = moment.utc(max); const duration = moment.duration(to.valueOf() - from.valueOf(), 'ms'); - let bucketSize = calculateAuto.near(100, duration).asSeconds(); - if (bucketSize < 1) bucketSize = 1; // don't go too small + + return calculateAuto.near(100, duration).asSeconds(); +}; + +export default (req, interval, capabilities) => { + const bucketSize = getTimeRangeBucketSize(req.payload.timerange); let intervalString = `${bucketSize}s`; - const gteAutoMatch = interval && interval.match(GTE_INTERVAL_RE); + const gteAutoMatch = Boolean(interval) && interval.match(GTE_INTERVAL_RE); + if (gteAutoMatch) { - const intervalStringMatch = gteAutoMatch[1].match(INTERVAL_STRING_RE); - const gteBucketSize = Number(intervalStringMatch[1]) * unitToSeconds(intervalStringMatch[2]); - if (gteBucketSize >= bucketSize) { - return { - bucketSize: gteBucketSize, - intervalString: gteAutoMatch[1] - }; + const bucketData = calculateBucketData(gteAutoMatch[1], capabilities); + + if (bucketData.bucketSize >= bucketSize) { + return bucketData; } } const matches = interval && interval.match(INTERVAL_STRING_RE); + if (matches) { - bucketSize = Number(matches[1]) * unitToSeconds(matches[2]); intervalString = interval; } - return { bucketSize, intervalString }; + return calculateBucketData(intervalString, capabilities); }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js index 36636d8c57fdc..839383b7d3695 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_es_shard_timeout.js @@ -17,6 +17,6 @@ * under the License. */ -export default function getEsShardTimeout(req) { +export function getEsShardTimeout(req) { return req.server.config().get('elasticsearch.shardTimeout'); } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js index 99734f7f131c3..75d1b4d565fe5 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/get_splits.js @@ -23,12 +23,20 @@ import _ from 'lodash'; import getLastMetric from './get_last_metric'; import getSplitColors from './get_split_colors'; import { formatKey } from './format_key'; -export default function getSplits(resp, panel, series) { - const meta = _.get(resp, `aggregations.${series.id}.meta`); + +const getTimeSeries = (resp, series) => + _.get(resp, `aggregations.timeseries`) || + _.get(resp, `aggregations.${series.id}.timeseries`); + +export default function getSplits(resp, panel, series, meta) { + if (!meta) { + meta = _.get(resp, `aggregations.${series.id}.meta`); + } + const color = new Color(series.color); const metric = getLastMetric(series); - if (_.has(resp, `aggregations.${series.id}.buckets`)) { - const buckets = _.get(resp, `aggregations.${series.id}.buckets`); + const buckets = _.get(resp, `aggregations.${series.id}.buckets`); + if (buckets) { if (Array.isArray(buckets)) { const size = buckets.length; const colors = getSplitColors(series.color, size, series.split_color_mode); @@ -41,7 +49,7 @@ export default function getSplits(resp, panel, series) { }); } - if(series.split_mode === 'filters' && _.isPlainObject(buckets)) { + if (series.split_mode === 'filters' && _.isPlainObject(buckets)) { return series.split_filters.map(filter => { const bucket = _.get(resp, `aggregations.${series.id}.buckets.${filter.id}`); bucket.id = `${series.id}:${filter.id}`; @@ -54,9 +62,10 @@ export default function getSplits(resp, panel, series) { } } - const timeseries = _.get(resp, `aggregations.${series.id}.timeseries`); + const timeseries = getTimeSeries(resp, series); + const mergeObj = { - timeseries + timeseries, }; series.metrics .filter(m => /_bucket/.test(m.type)) @@ -69,8 +78,8 @@ export default function getSplits(resp, panel, series) { label: series.label || calculateLabel(metric, series.metrics), color: color.string(), ...mergeObj, - meta - } + meta, + }, ]; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js index f5159d8b2bf43..01df1a6f54acd 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/index.js @@ -22,7 +22,6 @@ import getAggValue from './get_agg_value'; import getBucketSize from './get_bucket_size'; import getBucketPath from './get_buckets_path'; import getDefaultDecoration from './get_default_decoration'; -import getEsShardTimeout from './get_es_shard_timeout'; import getLastMetric from './get_last_metric'; import getSiblingAggValue from './get_sibling_agg_value'; import getSplits from './get_splits'; @@ -37,7 +36,6 @@ export default { getBucketSize, getBucketPath, getDefaultDecoration, - getEsShardTimeout, getLastMetric, getSiblingAggValue, getSplits, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js new file mode 100644 index 0000000000000..1143291141dc4 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/metric_types.js @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const EXTENDED_STATS_TYPES = [ + 'std_deviation', + 'variance', + 'sum_of_squares' +]; + +export const METRIC_TYPES = { + PERCENTILE: 'percentile', + PERCENTILE_RANK: 'percentile_rank', + TOP_HIT: 'top_hit', + COUNT: 'count' +}; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js index dbbbd9a68f1d7..4494883c44edc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/helpers/unit_to_seconds.js @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { INTERVAL_STRING_RE } from '../../../../common/interval_regexp'; const units = { ms: 0.001, @@ -23,11 +24,33 @@ const units = { m: 60, h: 3600, d: 86400, - w: (86400) * 7, // Hum... might be wrong - M: (86400) * 30, // this too... 29,30,31? - y: (86400) * 356 // Leap year? + w: 86400 * 7, // Hum... might be wrong + M: 86400 * 7 * 4, // this too... 29,30,31? + y: 86400 * 7 * 4 * 12, // Leap year? +}; + +export const parseInterval = (intervalString) => { + let value; + let unit; + + if (intervalString) { + const matches = intervalString.match(INTERVAL_STRING_RE); + + value = Number(matches[1]); + unit = matches[2]; + } + + return { value, unit }; +}; + +export const convertIntervalToUnit = (intervalString, unit) => { + const parsedInterval = parseInterval(intervalString); + const value = Number((parsedInterval.value * units[parsedInterval.unit] / units[unit]).toFixed(2)); + + return { value, unit }; }; export default (unit) => { return units[unit]; }; + diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js index 323d5b4911b27..05657842a5e2a 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/date_histogram.js @@ -20,12 +20,13 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import getTimerange from '../../helpers/get_timerange'; -export default function dateHistogram(req, panel, annotation) { +export default function dateHistogram(req, panel, annotation, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const timeField = annotation.time_field; - const { bucketSize, intervalString } = getBucketSize(req, 'auto'); + const { bucketSize, intervalString } = getBucketSize(req, 'auto', capabilities); const { from, to } = getTimerange(req); - const { timezone } = req.payload.timerange; + const timezone = capabilities.searchTimezone; + _.set(doc, `aggs.${annotation.id}.date_histogram`, { field: timeField, interval: intervalString, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js index 2e4f765b21321..882f35a2e0ee2 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/annotations/query.js @@ -21,10 +21,10 @@ import getBucketSize from '../../helpers/get_bucket_size'; import getTimerange from '../../helpers/get_timerange'; import { buildEsQuery } from '@kbn/es-query'; -export default function query(req, panel, annotation, esQueryConfig, indexPattern) { +export default function query(req, panel, annotation, esQueryConfig, indexPattern, capabilities) { return next => doc => { const timeField = annotation.time_field; - const { bucketSize } = getBucketSize(req, 'auto'); + const { bucketSize } = getBucketSize(req, 'auto', capabilities); const { from, to } = getTimerange(req); doc.size = 0; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js index 3d77d98e56950..34bcf090fecf2 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/__tests__/date_histogram.js @@ -20,12 +20,17 @@ import dateHistogram from '../date_histogram'; import { expect } from 'chai'; import sinon from 'sinon'; +import { DefaultSearchCapabilities } from '../../../../search_strategies/default_search_capabilities'; describe('dateHistogram(req, panel, series)', () => { let panel; let series; let req; + let capabilities; + let config; + let indexPatternObject; + beforeEach(() => { req = { payload: { @@ -42,17 +47,23 @@ describe('dateHistogram(req, panel, series)', () => { interval: '10s' }; series = { id: 'test' }; + config = { + allowLeadingWildcards: true, + queryStringOptions: {}, + }; + indexPatternObject = {}; + capabilities = new DefaultSearchCapabilities(req, true); }); it('calls next when finished', () => { const next = sinon.spy(); - dateHistogram(req, panel, series)(next)({}); + dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(next.calledOnce).to.equal(true); }); it('returns valid date histogram', () => { const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -73,7 +84,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 10, intervalString: '10s', - timeField: '@timestamp' + timeField: '@timestamp', + seriesId: 'test' } } } @@ -83,7 +95,7 @@ describe('dateHistogram(req, panel, series)', () => { it('returns valid date histogram (offset by 1h)', () => { series.offset_time = '1h'; const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -104,7 +116,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 10, intervalString: '10s', - timeField: '@timestamp' + timeField: '@timestamp', + seriesId: 'test' } } } @@ -117,7 +130,7 @@ describe('dateHistogram(req, panel, series)', () => { series.series_time_field = 'timestamp'; series.series_interval = '20s'; const next = doc => doc; - const doc = dateHistogram(req, panel, series)(next)({}); + const doc = dateHistogram(req, panel, series, config, indexPatternObject, capabilities)(next)({}); expect(doc).to.eql({ aggs: { test: { @@ -138,7 +151,8 @@ describe('dateHistogram(req, panel, series)', () => { meta: { bucketSize: 20, intervalString: '20s', - timeField: 'timestamp' + timeField: 'timestamp', + seriesId: 'test' } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js index b760ef20da053..c04684aab7e4f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/date_histogram.js @@ -21,12 +21,12 @@ import getBucketSize from '../../helpers/get_bucket_size'; import offsetTime from '../../offset_time'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; import { set } from 'lodash'; -export default function dateHistogram(req, panel, series) { +export default function dateHistogram(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { timeField, interval } = getIntervalAndTimefield(panel, series); - const { bucketSize, intervalString } = getBucketSize(req, interval); + const { bucketSize, intervalString } = getBucketSize(req, interval, capabilities); const { from, to } = offsetTime(req, series.offset_time); - const { timezone } = req.payload.timerange; + const timezone = capabilities.searchTimezone; set(doc, `aggs.${series.id}.aggs.timeseries.date_histogram`, { field: timeField, @@ -41,7 +41,8 @@ export default function dateHistogram(req, panel, series) { set(doc, `aggs.${series.id}.meta`, { timeField, intervalString, - bucketSize + bucketSize, + seriesId: series.id, }); return next(doc); }; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js index 8008a63e10220..02e084bc779a1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/index.js @@ -26,6 +26,7 @@ import dateHistogram from './date_histogram'; import metricBuckets from './metric_buckets'; import siblingBuckets from './sibling_buckets'; import filterRatios from './filter_ratios'; +import normalizeQuery from './normalize_query'; export default [ query, @@ -36,5 +37,6 @@ export default [ dateHistogram, metricBuckets, siblingBuckets, - filterRatios + filterRatios, + normalizeQuery ]; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js index ed96bebc9c68c..19aad8f37d825 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/metric_buckets.js @@ -21,14 +21,14 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function metricBuckets(req, panel, series) { +export default function metricBuckets(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { interval } = getIntervalAndTimefield(panel, series); const { intervalString - } = getBucketSize(req, interval); + } = getBucketSize(req, interval, capabilities); series.metrics .filter(row => !/_bucket$/.test(row.type) && !/^series/.test(row.type)) .forEach(metric => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js new file mode 100644 index 0000000000000..df2a829b90024 --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/normalize_query.js @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +const { set, get, isEmpty } = require('lodash'); + +const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); + +/* For grouping by the 'Everything', the splitByEverything request processor + * creates fake .filter.match_all filter (see split_by_everything.js) to simplify the request processors code. + * But “filters” are not supported by all of available search strategies (e.g. Rollup search). + * This method removes that aggregation. + */ +function removeEmptyTopLevelAggregation(doc, series) { + const filter = get(doc, `aggs.${series.id}.filter`); + + if (isEmptyFilter(filter)) { + const meta = get(doc, `aggs.${series.id}.meta`); + set(doc, `aggs`, doc.aggs[series.id].aggs); + set(doc, `aggs.timeseries.meta`, meta); + } + + return doc; +} + +/* Last query handler in the chain. You can use this handler + * as the last place where you can modify the "doc" (request body) object before sending it to ES. + */ +export default function normalizeQuery(req, panel, series) { + return next => doc => { + return next(removeEmptyTopLevelAggregation(doc, series)); + }; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js index 9f2bc1d4cba0f..351ff2bf27f62 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/series/sibling_buckets.js @@ -21,14 +21,14 @@ import _ from 'lodash'; import getBucketSize from '../../helpers/get_bucket_size'; import bucketTransform from '../../helpers/bucket_transform'; import getIntervalAndTimefield from '../../get_interval_and_timefield'; -export default function siblingBuckets(req, panel, series) { +export default function siblingBuckets(req, panel, series, esQueryConfig, indexPatternObject, capabilities) { return next => doc => { const { interval } = getIntervalAndTimefield(panel, series); const { bucketSize - } = getBucketSize(req, interval); + } = getBucketSize(req, interval, capabilities); series.metrics .filter(row => /_bucket$/.test(row.type)) .forEach(metric => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js index dcafbe39cd153..ef4bf22e14c0f 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/filter_ratios.js @@ -23,7 +23,7 @@ import bucketTransform from '../../helpers/bucket_transform'; import _ from 'lodash'; import { calculateAggRoot } from './calculate_agg_root'; export default function ratios(req, panel) { - return () => doc => { + return next => doc => { panel.series.forEach(column => { const aggRoot = calculateAggRoot(doc, column); if (column.metrics.some(filter)) { @@ -63,6 +63,6 @@ export default function ratios(req, panel) { }); } }); - return doc; + return next(doc); }; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js index f2359e5bfffde..0e160e8333c5c 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/index.js @@ -25,6 +25,7 @@ import dateHistogram from './date_histogram'; import metricBuckets from './metric_buckets'; import siblingBuckets from './sibling_buckets'; import filterRatios from './filter_ratios'; +import normalizeQuery from './normalize_query'; export default [ query, @@ -34,5 +35,6 @@ export default [ dateHistogram, metricBuckets, siblingBuckets, - filterRatios + filterRatios, + normalizeQuery ]; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js new file mode 100644 index 0000000000000..2d8eb281858bf --- /dev/null +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/request_processors/table/normalize_query.js @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +const { set, get, isEmpty, forEach } = require('lodash'); + +const isEmptyFilter = (filter = {}) => Boolean(filter.match_all) && isEmpty(filter.match_all); + +/* Last query handler in the chain. You can use this handler + * as the last place where you can modify the "doc" (request body) object before sending it to ES. + */ +export default function normalizeQuery() { + return () => doc => { + const series = get(doc, 'aggs.pivot.aggs'); + const normalizedSeries = {}; + + forEach(series, (value, seriesId) => { + const filter = get(value, `filter`); + + if (isEmptyFilter(filter)) { + const agg = get(value, 'aggs.timeseries'); + const meta = { + ...get(value, 'meta'), + seriesId + }; + set(normalizedSeries, `${seriesId}`, agg); + set(normalizedSeries, `${seriesId}.meta`, meta); + } else { + set(normalizedSeries, `${seriesId}`, value); + } + }); + + set(doc, 'aggs.pivot.aggs', normalizedSeries); + + return doc; + }; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js index 0fa97bfb109e1..ada43f1e35cf5 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js @@ -25,7 +25,7 @@ import getSplits from '../../helpers/get_splits'; import mapBucket from '../../helpers/map_bucket'; import { evaluate } from 'tinymath'; -export function mathAgg(resp, panel, series) { +export function mathAgg(resp, panel, series, meta) { return next => results => { const mathMetric = last(series.metrics); if (mathMetric.type !== 'math') return next(results); @@ -38,7 +38,7 @@ export function mathAgg(resp, panel, series) { return true; }); const decoration = getDefaultDecoration(series); - const splits = getSplits(resp, panel, series); + const splits = getSplits(resp, panel, series, meta); const mathSeries = splits.map(split => { if (mathMetric.variables.length) { // Gather the data for the splits. The data will either be a sibling agg or diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js index 79534452665a9..e016aad1915b1 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/percentile.js @@ -22,12 +22,12 @@ import getAggValue from '../../helpers/get_agg_value'; import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; -export default function percentile(resp, panel, series) { +export default function percentile(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type !== 'percentile') return next(results); - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { metric.percentiles.forEach(percentile => { const label = (split.label) + ` (${percentile.value})`; const data = split.timeseries.buckets.map(bucket => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js index 7a52ef26e72ba..5d4e6991fe524 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_bands.js @@ -21,11 +21,11 @@ import _ from 'lodash'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import mapBucket from '../../helpers/map_bucket'; -export default function stdDeviationBands(resp, panel, series) { +export default function stdDeviationBands(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type === 'std_deviation' && metric.mode === 'band') { - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const upper = split.timeseries.buckets.map(mapBucket(_.assign({}, metric, { mode: 'upper' }))); const lower = split.timeseries.buckets.map(mapBucket(_.assign({}, metric, { mode: 'lower' }))); results.push({ diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js index db26760e30fde..45f5a0eb6bf73 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_deviation_sibling.js @@ -21,11 +21,11 @@ import _ from 'lodash'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import getSiblingAggValue from '../../helpers/get_sibling_agg_value'; -export default function stdDeviationSibling(resp, panel, series) { +export default function stdDeviationSibling(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.mode === 'band' && metric.type === 'std_deviation_bucket') { - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const mapBucketByMode = (mode) => { return bucket => { diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js index bc3cf9dab6338..06030a09f2510 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_metric.js @@ -21,7 +21,7 @@ import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import mapBucket from '../../helpers/map_bucket'; -export default function stdMetric(resp, panel, series) { +export default function stdMetric(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); if (metric.type === 'std_deviation' && metric.mode === 'band') { @@ -32,7 +32,7 @@ export default function stdMetric(resp, panel, series) { } if (/_bucket$/.test(metric.type)) return next(results); const decoration = getDefaultDecoration(series); - getSplits(resp, panel, series).forEach(split => { + getSplits(resp, panel, series, meta).forEach(split => { const data = split.timeseries.buckets.map(mapBucket(metric)); results.push({ id: `${split.id}`, diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js index eac35d459b4a1..c3784e96e7390 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/response_processors/series/std_sibling.js @@ -21,7 +21,7 @@ import getDefaultDecoration from '../../helpers/get_default_decoration'; import getSplits from '../../helpers/get_splits'; import getLastMetric from '../../helpers/get_last_metric'; import getSiblingAggValue from '../../helpers/get_sibling_agg_value'; -export default function stdSibling(resp, panel, series) { +export default function stdSibling(resp, panel, series, meta) { return next => results => { const metric = getLastMetric(series); @@ -29,7 +29,7 @@ export default function stdSibling(resp, panel, series) { if (metric.type === 'std_deviation_bucket' && metric.mode === 'band') return next(results); const decoration = getDefaultDecoration(series); - getSplits(resp, panel, series).forEach((split) => { + getSplits(resp, panel, series, meta).forEach((split) => { const data = split.timeseries.buckets.map(bucket => { return [bucket.key, getSiblingAggValue(split, metric)]; }); diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js index cb944bb8363fa..195ae5b01c0f9 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/__tests__/build_request_body.js @@ -75,18 +75,26 @@ const body = JSON.parse(` } `); -import buildRequestBody from '../build_request_body'; +import sinon from 'sinon'; import { expect } from 'chai'; +import { buildRequestBody } from '../build_request_body'; describe('buildRequestBody(req)', () => { it('returns a valid body', () => { const panel = body.panels[0]; const series = panel.series[0]; + const getValidTimeInterval = sinon.spy(() => '10s'); + const capabilities = { + searchTimezone: 'UTC', + getValidTimeInterval + }; const config = { allowLeadingWildcards: true, queryStringOptions: {}, }; - const doc = buildRequestBody({ payload: body }, panel, series, config); + const indexPatternObject = {}; + const doc = buildRequestBody({ payload: body }, panel, series, config, indexPatternObject, capabilities); + expect(doc).to.eql({ size: 0, query: { @@ -121,42 +129,36 @@ describe('buildRequestBody(req)', () => { } }, aggs: { - 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7': { - filter: { - match_all: {} - }, - meta: { - timeField: '@timestamp', - bucketSize: 10, - intervalString: '10s' - }, + timeseries: { aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - interval: '10s', - min_doc_count: 0, - time_zone: 'UTC', - extended_bounds: { - min: 1485463055881, - max: 1485463955881 - } - }, - aggs: { - 'c9b5f9c1-e403-11e6-be91-6f7688e9fac7': { - bucket_script: { - buckets_path: { - count: '_count' - }, - script: { - source: 'count * 1', - lang: 'expression' - }, - gap_policy: 'skip' - } + 'c9b5f9c1-e403-11e6-be91-6f7688e9fac7': { + bucket_script: { + buckets_path: { + count: '_count' + }, + gap_policy: 'skip', + script: { + lang: 'expression', + source: 'count * 1' } } } + }, + date_histogram: { + extended_bounds: { + max: 1485463955881, + min: 1485463055881 + }, + field: '@timestamp', + interval: '10s', + min_doc_count: 0, + time_zone: 'UTC' + }, + meta: { + bucketSize: 10, + intervalString: '10s', + seriesId: 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7', + timeField: '@timestamp' } } } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js index c25127b68a6bc..a5b724e11ef55 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/build_request_body.js @@ -20,10 +20,21 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/series'; -function buildRequestBody(req, panel, series, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, series, esQueryConfig, indexPattern); +/** + * Builds series request body + * + * @param {...args}: [ + * req: {Object} - a request object, + * panel: {Object} - a panel object, + * series: {Object} - an series object, + * esQueryConfig: {Object} - es query config object, + * indexPatternObject: {Object} - an index pattern object, + * capabilities: {Object} - a search capabilities object + * ] + * @returns {Object} doc - processed body + */ +export function buildRequestBody(...args) { + const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } - -export default buildRequestBody; diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js index d75192f01de2f..2f6a79bba10dc 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/get_request_params.js @@ -16,26 +16,28 @@ * specific language governing permissions and limitations * under the License. */ - -import buildRequestBody from './build_request_body'; +import { buildRequestBody } from './build_request_body'; +import { getEsShardTimeout } from '../helpers/get_es_shard_timeout'; import { getIndexPatternObject } from '../helpers/get_index_pattern'; -import getEsShardTimeout from '../helpers/get_es_shard_timeout'; -export default async (req, panel, series, esQueryConfig) => { +export async function getSeriesRequestParams(req, panel, series, esQueryConfig, capabilities) { + const bodies = []; const indexPattern = series.override_index_pattern && series.series_index_pattern || panel.index_pattern; const { indexPatternObject, indexPatternString } = await getIndexPatternObject(req, indexPattern); - const request = buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject); + const request = buildRequestBody(req, panel, series, esQueryConfig, indexPatternObject, capabilities); const esShardTimeout = getEsShardTimeout(req); + if (capabilities.batchRequestsSupport) { + bodies.push({ + index: indexPatternString, + }); + } + if (esShardTimeout > 0) { request.timeout = `${esShardTimeout}ms`; } - return [ - { - index: indexPatternString, - ignoreUnavailable: true, - }, - request, - ]; -}; + bodies.push(request); + + return bodies; +} diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js index f732db82f45ed..cd108139fffe7 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/series/handle_response_body.js @@ -44,9 +44,11 @@ export default function handleResponseBody(panel) { }) ); } - const seriesId = keys[0]; - const series = panel.series.find(s => s.id === seriesId); - const processor = buildProcessorFunction(processors, resp, panel, series); + const [ seriesId ] = keys; + const meta = get(resp, `aggregations.${seriesId}.meta`, {}); + const series = panel.series.find(s => s.id === (meta.seriesId || seriesId)); + const processor = buildProcessorFunction(processors, resp, panel, series, meta); + return processor([]); }; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js index 7e509a5ebf2e5..349863d9cb6c0 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/build_request_body.js @@ -20,8 +20,8 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../request_processors/table'; -function buildRequestBody(req, panel, esQueryConfig, indexPattern) { - const processor = buildProcessorFunction(processors, req, panel, esQueryConfig, indexPattern); +function buildRequestBody(...args) { + const processor = buildProcessorFunction(processors, ...args); const doc = processor({}); return doc; } diff --git a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js index a42b0f4eefecc..fcbae1518924e 100644 --- a/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js +++ b/src/legacy/core_plugins/metrics/server/lib/vis_data/table/process_bucket.js @@ -21,10 +21,21 @@ import buildProcessorFunction from '../build_processor_function'; import processors from '../response_processors/table'; import getLastValue from '../../../../common/get_last_value'; import regression from 'regression'; -import { first, get } from 'lodash'; +import { first, get, set } from 'lodash'; export default function processBucket(panel) { return bucket => { const series = panel.series.map(series => { + const timeseries = get(bucket, `${series.id}.timeseries`); + const buckets = get(bucket, `${series.id}.buckets`); + + if (!timeseries && buckets) { + const meta = get(bucket, `${series.id}.meta`); + const timeseries = { + buckets: get(bucket, `${series.id}.buckets`) + }; + set(bucket, series.id, { meta, timeseries }); + } + const processor = buildProcessorFunction(processors, bucket, panel, series); const result = first(processor([])); if (!result) return null; diff --git a/src/legacy/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap b/src/legacy/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap index 61cb0ab4f07bc..c86d5eb85ad3b 100644 --- a/src/legacy/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap +++ b/src/legacy/ui/public/visualize/loader/pipeline_helpers/__snapshots__/build_pipeline.test.js.snap @@ -8,7 +8,7 @@ exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunct exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metric function without buckets 1`] = `"kibana_metric visConfig='{\\"metric\\":{\\"metrics\\":[0,1]}}' "`; -exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metrics/tsvb function 1`] = `"tsvb params='{\\"foo\\":\\"bar\\"}' "`; +exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles metrics/tsvb function 1`] = `"tsvb params='{\\"foo\\":\\"bar\\"}' uiState='{}' "`; exports[`visualize loader pipeline helpers: build pipeline buildPipelineVisFunction handles pie function 1`] = `"kibana_pie visConfig='{\\"dimensions\\":{\\"metric\\":0,\\"buckets\\":[1,2]}}' "`; diff --git a/src/legacy/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts b/src/legacy/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts index f2f13c91f02fb..6398029879041 100644 --- a/src/legacy/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts +++ b/src/legacy/ui/public/visualize/loader/pipeline_helpers/build_pipeline.ts @@ -49,7 +49,7 @@ interface Schemas { [key: string]: any[] | undefined; } -type buildVisFunction = (visState: VisState, schemas: Schemas) => string; +type buildVisFunction = (visState: VisState, schemas: Schemas, uiState: any) => string; type buildVisConfigFunction = (visState: Vis, schemas: Schemas) => VisState; interface BuildPipelineVisFunction { @@ -206,8 +206,11 @@ export const buildPipelineVisFunction: BuildPipelineVisFunction = { input_control_vis: visState => { return `input_control_vis ${prepareJson('visConfig', visState.params)}`; }, - metrics: visState => { - return `tsvb ${prepareJson('params', visState.params)}`; + metrics: (visState, schemas, uiState = {}) => { + const paramsJson = prepareJson('params', visState.params); + const uiStateJson = prepareJson('uiState', uiState); + + return `tsvb ${paramsJson} ${uiStateJson}`; }, timelion: visState => { const expression = prepareString('expression', visState.params.expression); @@ -347,6 +350,7 @@ export const buildPipeline = ( const query = searchSource.getField('query'); const filters = searchSource.getField('filter'); const visState = vis.getCurrentState(); + const uiState = vis.getUiState(); // context let pipeline = `kibana | kibana_context `; @@ -372,7 +376,7 @@ export const buildPipeline = ( const schemas = getSchemas(vis, params.timeRange); if (buildPipelineVisFunction[vis.type.name]) { - pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas); + pipeline += buildPipelineVisFunction[vis.type.name](visState, schemas, uiState); } else if (vislibCharts.includes(vis.type.name)) { const visConfig = visState.params; visConfig.dimensions = buildVislibDimensions(vis, params.timeRange); diff --git a/x-pack/plugins/rollup/index.js b/x-pack/plugins/rollup/index.js index 02095c97d9379..663cc92f38c32 100644 --- a/x-pack/plugins/rollup/index.js +++ b/x-pack/plugins/rollup/index.js @@ -8,6 +8,7 @@ import { resolve } from 'path'; import { PLUGIN } from './common'; import { registerLicenseChecker } from './server/lib/register_license_checker'; import { rollupDataEnricher } from './rollup_data_enricher'; +import { registerRollupSearchStrategy } from './server/lib/search_strategies'; import { registerIndicesRoute, registerFieldsForWildcardRoute, @@ -52,6 +53,8 @@ export function rollup(kibana) { ) { server.plugins.index_management.addIndexManagementDataEnricher(rollupDataEnricher); } + + registerRollupSearchStrategy(this.kbnServer, server); } }); } diff --git a/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js new file mode 100644 index 0000000000000..bbad5d9e4e48e --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/merge_capabilities_with_fields.js @@ -0,0 +1,67 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ + +// Merge rollup capabilities information with field information + +export const mergeCapabilitiesWithFields = (rollupIndexCapabilities, fieldsFromFieldCapsApi, previousFields = []) => { + const rollupFields = [...previousFields]; + const rollupFieldNames = []; + + Object.keys(rollupIndexCapabilities).forEach(agg => { + + // Field names of the aggregation + const fields = Object.keys(rollupIndexCapabilities[agg]); + + // Default field information + const defaultField = { + name: null, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }; + + // Date histogram agg only ever has one field defined, let date type overwrite a + // previous type if defined (such as number from max and min aggs). + if(agg === 'date_histogram') { + const timeFieldName = fields[0]; + const fieldCapsKey = `${timeFieldName}.${agg}.timestamp`; + const newField = { + ...fieldsFromFieldCapsApi[fieldCapsKey], + ...defaultField, + name: timeFieldName, + }; + const existingField = rollupFields.find(field => field.name === timeFieldName); + + if(existingField) { + Object.assign(existingField, newField); + } else { + rollupFieldNames.push(timeFieldName); + rollupFields.push(newField); + } + } + // For all other aggs, filter out ones that have already been added to the field list + // because the same field can be part of multiple aggregations, but end consumption + // doesn't differentiate fields based on their aggregation abilities. + else { + rollupFields.push( + ...fields + .filter(field => !rollupFieldNames.includes(field)) + .map(field => { + // Expand each field into object format that end consumption expects. + const fieldCapsKey = `${field}.${agg}.value`; + rollupFieldNames.push(field); + return { + ...fieldsFromFieldCapsApi[fieldCapsKey], + ...defaultField, + name: field, + }; + }) + ); + } + }); + + return rollupFields; +}; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/index.js b/x-pack/plugins/rollup/server/lib/search_strategies/index.js new file mode 100644 index 0000000000000..2d2711dfc932f --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/index.js @@ -0,0 +1,7 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ + +export { registerRollupSearchStrategy } from './register_rollup_search_strategy'; diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js new file mode 100644 index 0000000000000..a7efed9850fc2 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -0,0 +1,27 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { getRollupSearchStrategy } from './rollup_search_strategy'; +import { getRollupSearchRequest } from './rollup_search_request'; +import { getRollupSearchCapabilities } from './rollup_search_capabilities'; + +export const registerRollupSearchStrategy = (kbnServer, server) => kbnServer.afterPluginsInit(() => { + if (!server.plugins.metrics) { + return; + } + + const { + addSearchStrategy, + AbstractSearchRequest, + AbstractSearchStrategy, + DefaultSearchCapabilities, + } = server.plugins.metrics; + + const RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); + const RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); + const RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); + + addSearchStrategy(new RollupSearchStrategy(server)); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js new file mode 100644 index 0000000000000..acd2d48c89706 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.test.js @@ -0,0 +1,52 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { registerRollupSearchStrategy } from './register_rollup_search_strategy'; + +describe('Register Rollup Search Strategy', () => { + let kbnServer; + let metrics; + + beforeEach(() => { + const afterPluginsInit = jest.fn((callback) => callback()); + + kbnServer = { + afterPluginsInit, + }; + + metrics = { + addSearchStrategy: jest.fn().mockName('addSearchStrategy'), + AbstractSearchRequest: jest.fn().mockName('AbstractSearchRequest'), + AbstractSearchStrategy: jest.fn().mockName('AbstractSearchStrategy'), + DefaultSearchCapabilities: jest.fn().mockName('DefaultSearchCapabilities'), + }; + }); + + test('should run initialization on "afterPluginsInit" hook', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: {}, + }); + + expect(kbnServer.afterPluginsInit).toHaveBeenCalled(); + }); + + test('should run initialization if metrics plugin available', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: { + metrics, + }, + }); + + expect(metrics.addSearchStrategy).toHaveBeenCalled(); + }); + + test('should not run initialization if metrics plugin unavailable', () => { + registerRollupSearchStrategy(kbnServer, { + plugins: {}, + }); + + expect(metrics.addSearchStrategy).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js new file mode 100644 index 0000000000000..3e26b5c9b8532 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.js @@ -0,0 +1,53 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { get } from 'lodash'; +import { unitsMap } from '@elastic/datemath'; + +const leastCommonInterval = (num = 0, base = 0) => Math.max(Math.ceil(num / base) * base, base); + +const getDateHistogramAggregation = (fieldsCapabilities, rollupIndex) => { + const dateHistogramField = fieldsCapabilities[rollupIndex].aggs.date_histogram; + + // there is also only one valid date_histogram field + return Object.values(dateHistogramField)[0]; +}; + +const isCalendarInterval = ({ unit, value }) => value === 1 && ['calendar', 'mixed'].includes(unitsMap[unit].type); + +export const getRollupSearchCapabilities = (DefaultSearchCapabilities) => + (class RollupSearchCapabilities extends DefaultSearchCapabilities { + constructor(req, batchRequestsSupport, fieldsCapabilities, rollupIndex) { + super(req, batchRequestsSupport, fieldsCapabilities); + + this.rollupIndex = rollupIndex; + this.dateHistogram = getDateHistogramAggregation(fieldsCapabilities, rollupIndex); + } + + get defaultTimeInterval() { + return get(this.dateHistogram, 'interval', null); + } + + get searchTimezone() { + return get(this.dateHistogram, 'time_zone', null); + } + + getValidTimeInterval(userIntervalString) { + const parsedRollupJobInterval = this.parseInterval(this.defaultTimeInterval); + const parsedUserInterval = this.parseInterval(userIntervalString); + + let { unit } = parsedRollupJobInterval; + let { value } = this.convertIntervalToUnit(userIntervalString, unit); + + if (isCalendarInterval(parsedRollupJobInterval) && isCalendarInterval(parsedUserInterval)) { + unit = value > 1 ? parsedUserInterval.unit : parsedRollupJobInterval.unit; + value = 1; + } else { + value = leastCommonInterval(value, parsedRollupJobInterval.value); + } + + return `${value}${unit}`; + } + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js new file mode 100644 index 0000000000000..39906783fa1fa --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_capabilities.test.js @@ -0,0 +1,147 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { getRollupSearchCapabilities } from './rollup_search_capabilities'; + +class DefaultSearchCapabilities { + constructor(request, batchRequestsSupport, fieldsCapabilities = {}) { + this.fieldsCapabilities = fieldsCapabilities; + this.parseInterval = jest.fn((interval) => interval); + } +} + +describe('Rollup Search Capabilities', () => { + const testTimeZone = 'time_zone'; + const testInterval = '10s'; + const rollupIndex = 'rollupIndex'; + const batchRequestsSupport = true; + const request = {}; + + let RollupSearchCapabilities; + let fieldsCapabilities; + let rollupSearchCaps; + + beforeEach(() => { + RollupSearchCapabilities = getRollupSearchCapabilities(DefaultSearchCapabilities); + fieldsCapabilities = { + [rollupIndex]: { + aggs: { + date_histogram: { + histogram_field: { + time_zone: testTimeZone, + interval: testInterval, + }, + }, + }, + }, + }; + + rollupSearchCaps = new RollupSearchCapabilities(request, batchRequestsSupport, fieldsCapabilities, rollupIndex); + }); + + test('should create instance of RollupSearchRequest', () => { + expect(rollupSearchCaps).toBeInstanceOf(DefaultSearchCapabilities); + expect(rollupSearchCaps.fieldsCapabilities).toBe(fieldsCapabilities); + expect(rollupSearchCaps.rollupIndex).toBe(rollupIndex); + }); + + test('should return the "timezone" for the rollup request', () => { + expect(rollupSearchCaps.searchTimezone).toBe(testTimeZone); + }); + + test('should return the default "interval" for the rollup request', () => { + expect(rollupSearchCaps.defaultTimeInterval).toBe(testInterval); + }); + + describe('getValidTimeInterval', () => { + let parsedDefaultInterval; + let parsedUserIntervalString; + let convertedIntervalIntoDefaultUnit; + + beforeEach(() => { + convertedIntervalIntoDefaultUnit = null; + + rollupSearchCaps.parseInterval = jest.fn() + .mockImplementationOnce(() => parsedDefaultInterval) + .mockImplementationOnce(() => parsedUserIntervalString); + rollupSearchCaps.convertIntervalToUnit = jest + .fn(() => convertedIntervalIntoDefaultUnit || parsedUserIntervalString); + }); + + test('should return 1w as common interval for 1w(user interval) and 1d(rollup interval) - calendar intervals', () => { + parsedDefaultInterval = { + value: 1, + unit: 'd', + }; + parsedUserIntervalString = { + value: 1, + unit: 'w', + }; + convertedIntervalIntoDefaultUnit = { + value: 7, + unit: 'd', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('1w'); + }); + + test('should return 1w as common interval for 1d(user interval) and 1w(rollup interval) - calendar intervals', () => { + parsedDefaultInterval = { + value: 1, + unit: 'w', + }; + parsedUserIntervalString = { + value: 1, + unit: 'd', + }; + convertedIntervalIntoDefaultUnit = { + value: 1 / 7, + unit: 'w', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('1w'); + }); + + test('should return 2y as common interval for 0.1y(user interval) and 2y(rollup interval) - fixed intervals', () => { + parsedDefaultInterval = { + value: 2, + unit: 'y', + }; + parsedUserIntervalString = { + value: 0.1, + unit: 'y', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('2y'); + }); + + test('should return 3h as common interval for 2h(user interval) and 3h(rollup interval) - fixed intervals', () => { + parsedDefaultInterval = { + value: 3, + unit: 'h', + }; + parsedUserIntervalString = { + value: 2, + unit: 'h', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('3h'); + }); + + test('should return 6m as common interval for 4m(user interval) and 3m(rollup interval) - fixed intervals', () => { + parsedDefaultInterval = { + value: 3, + unit: 'm', + }; + parsedUserIntervalString = { + value: 4, + unit: 'm', + }; + + expect(rollupSearchCaps.getValidTimeInterval()).toBe('6m'); + }); + }); + +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js new file mode 100644 index 0000000000000..6ec8d9a882168 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.js @@ -0,0 +1,21 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +const SEARCH_METHOD = 'rollup.search'; + +export const getRollupSearchRequest = (AbstractSearchRequest) => + (class RollupSearchRequest extends AbstractSearchRequest { + async search(options) { + const bodies = Array.isArray(options.body) ? options.body : [options.body]; + const requests = bodies + .map(body => this.callWithRequest(SEARCH_METHOD, { + body, + index: this.indexPattern, + rest_total_hits_as_int: true, + })); + + return await Promise.all(requests); + } + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js new file mode 100644 index 0000000000000..3c90cad4650e3 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_request.test.js @@ -0,0 +1,51 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { getRollupSearchRequest } from './rollup_search_request'; + +class AbstractSearchRequest { + indexPattern = 'indexPattern'; + callWithRequest = jest.fn(({ body }) => Promise.resolve(body)); +} + +describe('Rollup search request', () => { + let RollupSearchRequest; + + beforeEach(() => { + RollupSearchRequest = getRollupSearchRequest(AbstractSearchRequest); + }); + + test('should create instance of RollupSearchRequest', () => { + const rollupSearchRequest = new RollupSearchRequest(); + + expect(rollupSearchRequest).toBeInstanceOf(AbstractSearchRequest); + expect(rollupSearchRequest.search).toBeDefined(); + expect(rollupSearchRequest.indexPattern).toBeDefined(); + expect(rollupSearchRequest.callWithRequest).toBeDefined(); + }); + + test('should send one request for single search', async () => { + const rollupSearchRequest = new RollupSearchRequest(); + const body = 'body'; + + await rollupSearchRequest.search({ body }); + + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledTimes(1); + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledWith('rollup.search', { + body, + index: 'indexPattern', + rest_total_hits_as_int: true, + }); + }); + + test('should send multiple request for multi search', async () => { + const rollupSearchRequest = new RollupSearchRequest(); + const body = ['firstRequestBody', 'secondRequestBody']; + + await rollupSearchRequest.search({ body }); + + expect(rollupSearchRequest.callWithRequest).toHaveBeenCalledTimes(body.length); + }); +}); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js new file mode 100644 index 0000000000000..a38dc49cac915 --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.js @@ -0,0 +1,67 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { indexBy, isString } from 'lodash'; +import { callWithRequestFactory } from '../call_with_request_factory'; +import { mergeCapabilitiesWithFields } from '../merge_capabilities_with_fields'; +import { getCapabilitiesForRollupIndices } from '../map_capabilities'; + +const ROLLUP_INDEX_CAPABILITIES_METHOD = 'rollup.rollupIndexCapabilities'; +const batchRequestsSupport = false; + +const getRollupIndices = rollupData => Object.keys(rollupData); + +const isIndexPatternContainsWildcard = indexPattern => indexPattern.includes('*'); +const isIndexPatternValid = indexPattern => indexPattern && + isString(indexPattern) && !isIndexPatternContainsWildcard(indexPattern); + +export const getRollupSearchStrategy = (AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities) => + (class RollupSearchStrategy extends AbstractSearchStrategy { + name = 'rollup'; + + constructor(server) { + super(server, callWithRequestFactory, RollupSearchRequest); + } + + getRollupData(req, indexPattern) { + const callWithRequest = this.getCallWithRequestInstance(req); + + return callWithRequest(ROLLUP_INDEX_CAPABILITIES_METHOD, { + indexPattern, + }).catch(() => Promise.resolve({})); + } + + async checkForViability(req, indexPattern) { + let isViable = false; + let capabilities = null; + + if (isIndexPatternValid(indexPattern)) { + const rollupData = await this.getRollupData(req, indexPattern); + const rollupIndices = getRollupIndices(rollupData); + + isViable = rollupIndices.length === 1; + + if (isViable) { + const [rollupIndex] = rollupIndices; + const fieldsCapabilities = getCapabilitiesForRollupIndices(rollupData); + + capabilities = new RollupSearchCapabilities(req, batchRequestsSupport, fieldsCapabilities, rollupIndex); + } + } + + return { + isViable, + capabilities, + }; + } + + async getFieldsForWildcard(req, indexPattern, { fieldsCapabilities, rollupIndex }) { + const fields = await super.getFieldsForWildcard(req, indexPattern); + const fieldsFromFieldCapsApi = indexBy(fields, 'name'); + const rollupIndexCapabilities = fieldsCapabilities[rollupIndex].aggs; + + return mergeCapabilitiesWithFields(rollupIndexCapabilities, fieldsFromFieldCapsApi); + } + }); diff --git a/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js new file mode 100644 index 0000000000000..f13c2070f713a --- /dev/null +++ b/x-pack/plugins/rollup/server/lib/search_strategies/rollup_search_strategy.test.js @@ -0,0 +1,153 @@ +/* +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the Elastic License; +* you may not use this file except in compliance with the Elastic License. +*/ +import { getRollupSearchStrategy } from './rollup_search_strategy'; + +describe('Rollup Search Strategy', () => { + let RollupSearchStrategy; + let RollupSearchRequest; + let RollupSearchCapabilities; + let callWithRequest; + let rollupResolvedData; + + const server = 'server'; + const request = 'request'; + const indexPattern = 'indexPattern'; + + beforeEach(() => { + class AbstractSearchStrategy { + getCallWithRequestInstance = jest.fn(() => callWithRequest); + + getFieldsForWildcard() { + return [ + { + name: 'day_of_week.terms.value', + type: 'object', + searchable: false, + aggregatable: false, + }, + ]; + } + } + + RollupSearchRequest = jest.fn(); + RollupSearchCapabilities = jest.fn(() => 'capabilities'); + callWithRequest = jest.fn().mockImplementation(() => rollupResolvedData); + + RollupSearchStrategy = getRollupSearchStrategy(AbstractSearchStrategy, RollupSearchRequest, RollupSearchCapabilities); + }); + + test('should create instance of RollupSearchRequest', () => { + const rollupSearchStrategy = new RollupSearchStrategy(server); + + expect(rollupSearchStrategy.name).toBe('rollup'); + }); + + describe('checkForViability', () => { + let rollupSearchStrategy; + const rollupIndex = 'rollupIndex'; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + rollupSearchStrategy.getRollupData = jest.fn(() => ({ + [rollupIndex]: { + rollup_jobs: [{ + job_id: 'test', + rollup_index: rollupIndex, + index_pattern: 'kibana*', + fields: { + order_date: [{ + agg: 'date_histogram', + delay: '1m', + interval: '1m', + time_zone: 'UTC', + }], + day_of_week: [{ + agg: 'terms', + }], + }, + }], + }, + })); + }); + + test('isViable should be false for invalid index', async () => { + const result = await rollupSearchStrategy.checkForViability(request, null); + + expect(result).toEqual({ + isViable: false, + capabilities: null, + }); + }); + + test('should get RollupSearchCapabilities for valid rollup index ', async () => { + await rollupSearchStrategy.checkForViability(request, rollupIndex); + + expect(RollupSearchCapabilities).toHaveBeenCalled(); + }); + }); + + describe('getRollupData', () => { + let rollupSearchStrategy; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + }); + + test('should return rollup data', async () => { + rollupResolvedData = Promise.resolve('data'); + + const rollupData = await rollupSearchStrategy.getRollupData(request, indexPattern); + + expect(callWithRequest).toHaveBeenCalledWith('rollup.rollupIndexCapabilities', { indexPattern }); + expect(rollupSearchStrategy.getCallWithRequestInstance).toHaveBeenCalledWith(request); + expect(rollupData).toBe('data'); + }); + + test('should return empty object in case of exception', async () => { + rollupResolvedData = Promise.reject('data'); + + const rollupData = await rollupSearchStrategy.getRollupData(request, indexPattern); + + expect(rollupData).toEqual({}); + }); + }); + + describe('getFieldsForWildcard', () => { + let rollupSearchStrategy; + let fieldsCapabilities; + + const rollupIndex = 'rollupIndex'; + + beforeEach(() => { + rollupSearchStrategy = new RollupSearchStrategy(server); + fieldsCapabilities = { + [rollupIndex]: { + aggs: { + terms: { + day_of_week: { agg: 'terms' }, + }, + }, + }, + }; + }); + + test('should return fields for wildcard', async () => { + const fields = await rollupSearchStrategy.getFieldsForWildcard(request, indexPattern, + { fieldsCapabilities, rollupIndex }, + ); + + expect(fields).toEqual([ + { + aggregatable: true, + name: 'day_of_week', + readFromDocValues: true, + searchable: true, + type: 'object', + }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/rollup/server/routes/api/index_patterns.js b/x-pack/plugins/rollup/server/routes/api/index_patterns.js index 4b68d3abc0f60..7f760ac515896 100644 --- a/x-pack/plugins/rollup/server/routes/api/index_patterns.js +++ b/x-pack/plugins/rollup/server/routes/api/index_patterns.js @@ -10,6 +10,7 @@ import { wrapEsError, wrapUnknownError } from '../../lib/error_wrappers'; import { licensePreRoutingFactory } from'../../lib/license_pre_routing_factory'; import indexBy from 'lodash/collection/indexBy'; import { getCapabilitiesForRollupIndices } from '../../lib/map_capabilities'; +import { mergeCapabilitiesWithFields } from '../../lib/merge_capabilities_with_fields'; import querystring from 'querystring'; /** @@ -57,7 +58,6 @@ export function registerFieldsForWildcardRoute(server) { const callWithRequest = callWithRequestFactory(server, request); const rollupFields = []; - const rollupFieldNames = []; const fieldsFromFieldCapsApi = indexBy(fields, 'name'); const rollupIndexCapabilities = getCapabilitiesForRollupIndices(await callWithRequest('rollup.rollupIndexCapabilities', { indexPattern: rollupIndex @@ -66,62 +66,10 @@ export function registerFieldsForWildcardRoute(server) { // Keep meta fields metaFields.forEach(field => fieldsFromFieldCapsApi[field] && rollupFields.push(fieldsFromFieldCapsApi[field])); - // Merge rollup capabilities information with field information - Object.keys(rollupIndexCapabilities).forEach(agg => { - - // Field names of the aggregation - const fields = Object.keys(rollupIndexCapabilities[agg]); - - // Default field information - const defaultField = { - name: null, - searchable: true, - aggregatable: true, - readFromDocValues: true, - }; - - // Date histogram agg only ever has one field defined, let date type overwrite a - // previous type if defined (such as number from max and min aggs). - if(agg === 'date_histogram') { - const timeFieldName = fields[0]; - const fieldCapsKey = `${timeFieldName}.${agg}.timestamp`; - const newField = { - ...fieldsFromFieldCapsApi[fieldCapsKey], - ...defaultField, - name: timeFieldName, - }; - const existingField = rollupFields.find(field => field.name === timeFieldName); - - if(existingField) { - Object.assign(existingField, newField); - } else { - rollupFieldNames.push(timeFieldName); - rollupFields.push(newField); - } - } - // For all other aggs, filter out ones that have already been added to the field list - // because the same field can be part of multiple aggregations, but end consumption - // doesn't differentiate fields based on their aggregation abilities. - else { - rollupFields.push( - ...fields - .filter(field => !rollupFieldNames.includes(field)) - .map(field => { - // Expand each field into object format that end consumption expects. - const fieldCapsKey = `${field}.${agg}.value`; - rollupFieldNames.push(field); - return { - ...fieldsFromFieldCapsApi[fieldCapsKey], - ...defaultField, - name: field, - }; - }) - ); - } - }); + const mergedRollupFields = mergeCapabilitiesWithFields(rollupIndexCapabilities, fieldsFromFieldCapsApi, rollupFields); return { - fields: rollupFields + fields: [ ...rollupFields, ...mergedRollupFields ] }; } catch(err) { if (isEsError(err)) {