From 752609dc477c1e82817c0093c255bd14745a24b6 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 15 Jun 2021 21:34:01 -0700 Subject: [PATCH 01/14] skip flaky suite (#102282) --- x-pack/test/api_integration/apis/ml/modules/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/ml/modules/index.ts b/x-pack/test/api_integration/apis/ml/modules/index.ts index ab46c4f0333c8c..dae0044c47ccac 100644 --- a/x-pack/test/api_integration/apis/ml/modules/index.ts +++ b/x-pack/test/api_integration/apis/ml/modules/index.ts @@ -12,7 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const fleetPackages = ['apache-0.5.0', 'nginx-0.5.0']; - describe('modules', function () { + // Failing: See https://github.com/elastic/kibana/issues/102282 + describe.skip('modules', function () { before(async () => { for (const fleetPackage of fleetPackages) { await ml.testResources.installFleetPackage(fleetPackage); From 4cd073f65109f03b29dc5179dec120f5e141c07a Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 16 Jun 2021 08:11:32 +0100 Subject: [PATCH 02/14] Bug: ES query returning no records for d-rule alerts. (#102160) --- .../server/usage/detections/detection_rule_helpers.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts index ebcda694411355..8d5a2efc7fae11 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts @@ -177,6 +177,8 @@ export const updateDetectionRuleUsage = ( return updatedUsage; }; +const MAX_RESULTS_WINDOW = 10_000; // elasticsearch index.max_result_window default value + export const getDetectionRuleMetrics = async ( kibanaIndex: string, signalsIndex: string, @@ -189,14 +191,14 @@ export const getDetectionRuleMetrics = async ( filterPath: [], ignoreUnavailable: true, index: kibanaIndex, - size: 10_000, // elasticsearch index.max_result_window default value + size: MAX_RESULTS_WINDOW, }; try { const { body: ruleResults } = await esClient.search(ruleSearchOptions); const { body: detectionAlertsResp } = (await esClient.search({ index: `${signalsIndex}*`, - size: 0, + size: MAX_RESULTS_WINDOW, body: { aggs: { detectionAlerts: { @@ -224,7 +226,7 @@ export const getDetectionRuleMetrics = async ( type: 'cases-comments', fields: [], page: 1, - perPage: 10_000, + perPage: MAX_RESULTS_WINDOW, filter: 'cases-comments.attributes.type: alert', }); From f4d22b36fb300e8cbcee1606d1f7fde70b58b9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Wed, 16 Jun 2021 09:13:37 +0200 Subject: [PATCH 03/14] [ML] Adds popover help for ROC curve (#101893) Co-authored-by: Lisa Cawley --- .../evaluate_panel.tsx | 15 +---- .../roc_curve_help_popover.tsx | 55 +++++++++++++++++++ .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 4 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/roc_curve_help_popover.tsx diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx index bc1c9dbed1dcc7..086adcecd077a3 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/evaluate_panel.tsx @@ -17,7 +17,6 @@ import { EuiDataGridPopoverContents, EuiFlexGroup, EuiFlexItem, - EuiIconTip, EuiSpacer, EuiText, EuiTitle, @@ -51,6 +50,7 @@ import { isTrainingFilter } from './is_training_filter'; import { useRocCurve } from './use_roc_curve'; import { useConfusionMatrix } from './use_confusion_matrix'; import { MulticlassConfusionMatrixHelpPopover } from './confusion_matrix_help_popover'; +import { RocCurveHelpPopover } from './roc_curve_help_popover'; export interface EvaluatePanelProps { jobConfig: DataFrameAnalyticsConfig; @@ -409,7 +409,7 @@ export const EvaluatePanel: FC = ({ jobConfig, jobStatus, se {/* AUC ROC Chart */} - + = ({ jobConfig, jobStatus, se - + {Array.isArray(errorRocCurve) && ( diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/roc_curve_help_popover.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/roc_curve_help_popover.tsx new file mode 100644 index 00000000000000..f828cbabde8949 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/roc_curve_help_popover.tsx @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + HelpPopover, + HelpPopoverButton, +} from '../../../../../components/help_popover/help_popover'; + +export const RocCurveHelpPopover = () => { + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + + return ( + { + setIsPopoverOpen(!isPopoverOpen); + }} + /> + } + closePopover={() => setIsPopoverOpen(false)} + isOpen={isPopoverOpen} + title={i18n.translate('xpack.ml.dataframe.analytics.rocCurvePopoverTitle', { + defaultMessage: 'Receiver operating characteristic (ROC) curve', + })} + > +

+ +

+

+ +

+

+ +

+ + ); +}; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 1f5cbcd4d6c3cc..c67dd383a2ea2c 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14027,7 +14027,6 @@ "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionMeanRecallStat": "平均再現率", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionOverallAccuracyStat": "全体的な精度", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionOverallAccuracyTooltip": "合計予測数に対する正しいクラス予測数の比率。", - "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionRocInfoTooltip": "受信者操作特性 (ROC) 曲線は、異なる予測確率しきい値で分類プロセスのパフォーマンスを表すプロットです。", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionRocTitle": "受信者操作特性 (ROC) 曲線", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionTitle": "モデル評価", "xpack.ml.dataframe.analytics.classificationExploration.showActions": "アクションを表示", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 8006153cafb063..23cde5dd1fcff4 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14208,7 +14208,6 @@ "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionMeanRecallStat": "平均召回率", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionOverallAccuracyStat": "总体准确率", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionOverallAccuracyTooltip": "正确类预测数目与预测总数的比率。", - "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionRocInfoTooltip": "接受者操作特性 (ROC) 曲线是表示在不同预测概率阈值下分类过程的性能绘图。", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionRocTitle": "接受者操作特性 (ROC) 曲线", "xpack.ml.dataframe.analytics.classificationExploration.evaluateSectionTitle": "模型评估", "xpack.ml.dataframe.analytics.classificationExploration.generalizationDocsCount": "{docsCount, plural, other {个文档}}已评估", From f39bf9c985b139e29e9ac8d8c0b37ab1f96edbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Wed, 16 Jun 2021 09:35:08 +0200 Subject: [PATCH 04/14] [APM] More styles fixes related to the new page template (#102253) * [APM] Service map: Add border to panel * [APM] Metrics: Add border to panel * [APM] Service overview: Add border to panel * [APM] Transactions and charts: Add border to panel --- .../apm/public/components/app/service_map/index.tsx | 2 +- .../apm/public/components/app/service_metrics/index.tsx | 2 +- .../public/components/app/service_node_metrics/index.tsx | 6 ++++-- .../apm/public/components/app/service_overview/index.tsx | 8 ++++---- .../service_overview_instances_chart_and_table.tsx | 2 +- .../service_overview_throughput_chart.tsx | 2 +- .../transaction_details/WaterfallWithSummmary/index.tsx | 4 ++-- .../public/components/app/transaction_details/index.tsx | 6 ++++-- .../public/components/app/transaction_overview/index.tsx | 2 +- .../charts/instances_latency_distribution_chart/index.tsx | 2 +- .../shared/charts/transaction_breakdown_chart/index.tsx | 2 +- .../components/shared/charts/transaction_charts/index.tsx | 4 ++-- .../shared/charts/transaction_error_rate_chart/index.tsx | 2 +- 13 files changed, 24 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_map/index.tsx b/x-pack/plugins/apm/public/components/app/service_map/index.tsx index df8438c5c80a4a..582eafe7553af7 100644 --- a/x-pack/plugins/apm/public/components/app/service_map/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_map/index.tsx @@ -146,7 +146,7 @@ export function ServiceMap({ return ( <> - +
{data.charts.map((chart) => ( - + ) : ( - + + + )} @@ -171,7 +173,7 @@ export function ServiceNodeMetrics({ {data.charts.map((chart) => ( - + - + @@ -63,7 +63,7 @@ export function ServiceOverview({ serviceName }: ServiceOverviewProps) { - + @@ -84,7 +84,7 @@ export function ServiceOverview({ serviceName }: ServiceOverviewProps) { )} - + @@ -101,7 +101,7 @@ export function ServiceOverview({ serviceName }: ServiceOverviewProps) { {!isRumAgent && ( - + diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx index 8513e0835d373b..719409b0f97ffe 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx @@ -228,7 +228,7 @@ export function ServiceOverviewInstancesChartAndTable({ /> - + +

{i18n.translate('xpack.apm.serviceOverview.throughtputChartTitle', { diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/index.tsx index 3d3ce3262f13b5..6f5b95b103f6b7 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/index.tsx @@ -83,13 +83,13 @@ export function WaterfallWithSummmary({ /> ); - return {content}; + return {content}; } const entryTransaction = entryWaterfallTransaction.doc; return ( - + diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/index.tsx index b9508b2d303a26..3cac05ba2d96a8 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/index.tsx @@ -76,11 +76,13 @@ export function TransactionDetails() { return ( <> + +

{transactionName}

 - + @@ -88,7 +90,7 @@ export function TransactionDetails() { - + - +

Transactions

 diff --git a/x-pack/plugins/apm/public/components/shared/charts/instances_latency_distribution_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/instances_latency_distribution_chart/index.tsx index ce4f36ced79038..0ad4be17e35cbd 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/instances_latency_distribution_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/instances_latency_distribution_chart/index.tsx @@ -104,7 +104,7 @@ export function InstancesLatencyDistributionChart({ }; return ( - +

{i18n.translate('xpack.apm.instancesLatencyDistributionChartTitle', { diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/index.tsx index 978604c4c96ece..40c5e39589fb1f 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/index.tsx @@ -22,7 +22,7 @@ export function TransactionBreakdownChart({ const { timeseries } = data; return ( - + diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx index 3f868ae272e3a1..019a25b1e9ed37 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx @@ -37,13 +37,13 @@ export function TransactionCharts() { - + - + {i18n.translate( diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx index 7eceaf5ca8e5da..96cb7c49a67104 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx @@ -135,7 +135,7 @@ export function TransactionErrorRateChart({ ]; return ( - +

{i18n.translate('xpack.apm.errorRate', { From c9678f29e2df52ac19eb45067d6c072865389660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Wed, 16 Jun 2021 09:43:35 +0200 Subject: [PATCH 05/14] [UX] Update panels to use border not shadow (#102262) --- .../components/app/RumDashboard/ClientMetrics/index.tsx | 2 +- .../components/app/RumDashboard/ImpactfulMetrics/index.tsx | 2 +- .../components/app/RumDashboard/Panels/PageLoadAndViews.tsx | 4 ++-- .../components/app/RumDashboard/Panels/VisitorBreakdowns.tsx | 4 ++-- .../public/components/app/RumDashboard/UXMetrics/index.tsx | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/ClientMetrics/index.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/ClientMetrics/index.tsx index add6ac1b08b281..c525a71ea45891 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/ClientMetrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/ClientMetrics/index.tsx @@ -24,7 +24,7 @@ export function ClientMetrics() { } = useUrlParams(); return ( - + diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/ImpactfulMetrics/index.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/ImpactfulMetrics/index.tsx index 175b40f85d64b0..b696a46f59bd12 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/ImpactfulMetrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/ImpactfulMetrics/index.tsx @@ -11,7 +11,7 @@ import { JSErrors } from './JSErrors'; export function ImpactfulMetrics() { return ( - + diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/PageLoadAndViews.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/PageLoadAndViews.tsx index b51e2559b7f15d..9dd83fd1c8fd1f 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/PageLoadAndViews.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/PageLoadAndViews.tsx @@ -14,12 +14,12 @@ export function PageLoadAndViews() { return ( - + - + diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/VisitorBreakdowns.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/VisitorBreakdowns.tsx index 0433988ecfa21a..ff79feaa924f37 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/VisitorBreakdowns.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/Panels/VisitorBreakdowns.tsx @@ -14,12 +14,12 @@ export function VisitorBreakdownsPanel() { return ( - + - + diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/UXMetrics/index.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/UXMetrics/index.tsx index 44212fed989876..a665b6560c7e95 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/UXMetrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/UXMetrics/index.tsx @@ -62,7 +62,7 @@ export function UXMetrics() { ); return ( - + From b96914235e8112cc3cc4f3d7a197560f684a2d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Wed, 16 Jun 2021 09:43:58 +0200 Subject: [PATCH 06/14] [Observability] Add border to section container panels (#102259) --- .../observability/public/components/app/section/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability/public/components/app/section/index.tsx b/x-pack/plugins/observability/public/components/app/section/index.tsx index 191a1b2890ada3..cbe0c45bbd169c 100644 --- a/x-pack/plugins/observability/public/components/app/section/index.tsx +++ b/x-pack/plugins/observability/public/components/app/section/index.tsx @@ -25,7 +25,7 @@ interface Props { export function SectionContainer({ title, appLink, children, hasError }: Props) { const { core } = usePluginContext(); return ( - + Date: Wed, 16 Jun 2021 10:00:05 +0200 Subject: [PATCH 07/14] [Security Solution][Timeline] Fix User not able to scroll down and access Alert table on adding long content in Timeline's Description (#101486) * Add LineClamp component to timeline description * Truncate timeline description on timeline table * Fix StyledLineClamp styled component performance issue Read more: https://styled-components.com/docs/faqs#why-should-i-avoid-declaring-styled-components-in-the-render-method --- .../common/components/line_clamp/index.tsx | 29 ++++++++++++------- .../components/flyout/header/index.tsx | 11 +++---- .../timelines_table/common_columns.tsx | 14 +++++++-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/line_clamp/index.tsx b/x-pack/plugins/security_solution/public/common/components/line_clamp/index.tsx index 896b0ec5fd8df8..d8895490d1e0ff 100644 --- a/x-pack/plugins/security_solution/public/common/components/line_clamp/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/line_clamp/index.tsx @@ -13,15 +13,6 @@ import * as i18n from './translations'; const LINE_CLAMP = 3; const LINE_CLAMP_HEIGHT = 5.5; -const StyledLineClamp = styled.div` - display: -webkit-box; - -webkit-line-clamp: ${LINE_CLAMP}; - -webkit-box-orient: vertical; - overflow: hidden; - max-height: ${`${LINE_CLAMP_HEIGHT}em`}; - height: ${`${LINE_CLAMP_HEIGHT}em`}; -`; - const ReadMore = styled(EuiButtonEmpty)` span.euiButtonContent { padding: 0; @@ -35,7 +26,19 @@ const ExpandedContent = styled.div` overflow-y: auto; `; -const LineClampComponent: React.FC<{ content?: string | null }> = ({ content }) => { +const StyledLineClamp = styled.div<{ lineClampHeight: number }>` + display: -webkit-box; + -webkit-line-clamp: ${LINE_CLAMP}; + -webkit-box-orient: vertical; + overflow: hidden; + max-height: ${({ lineClampHeight }) => lineClampHeight}em; + height: ${({ lineClampHeight }) => lineClampHeight}em; +`; + +const LineClampComponent: React.FC<{ + content?: string | null; + lineClampHeight?: number; +}> = ({ content, lineClampHeight = LINE_CLAMP_HEIGHT }) => { const [isOverflow, setIsOverflow] = useState(null); const [isExpanded, setIsExpanded] = useState(null); const descriptionRef = useRef(null); @@ -71,7 +74,11 @@ const LineClampComponent: React.FC<{ content?: string | null }> = ({ content })

{content}

) : isOverflow == null || isOverflow === true ? ( - + {content} ) : ( diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx index da45579b347739..dd8cdb818cad75 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/header/index.tsx @@ -52,6 +52,7 @@ import * as i18n from './translations'; import * as commonI18n from '../../timeline/properties/translations'; import { getTimelineStatusByIdSelector } from './selectors'; import { TimelineKPIs } from './kpis'; +import { LineClamp } from '../../../../common/components/line_clamp'; // to hide side borders const StyledPanel = styled(EuiPanel)` @@ -206,13 +207,13 @@ const TimelineDescriptionComponent: React.FC = ({ timelineId (state) => (getTimeline(state, timelineId) ?? timelineDefaults).description ); - const content = useMemo(() => (description.length ? description : commonI18n.DESCRIPTION), [ - description, - ]); - return ( - {content} + {description.length ? ( + + ) : ( + commonI18n.DESCRIPTION + )} ); }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/timelines_table/common_columns.tsx b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/timelines_table/common_columns.tsx index 98d678a25b4c68..65963c96093209 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/timelines_table/common_columns.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/timelines_table/common_columns.tsx @@ -10,7 +10,7 @@ import { EuiButtonIcon, EuiLink } from '@elastic/eui'; import { omit } from 'lodash/fp'; import React from 'react'; - +import styled from 'styled-components'; import { ACTION_COLUMN_WIDTH } from './common_styles'; import { isUntitled } from '../helpers'; import { NotePreviews } from '../note_previews'; @@ -20,6 +20,14 @@ import { getEmptyTagValue } from '../../../../common/components/empty_value'; import { FormattedRelativePreferenceDate } from '../../../../common/components/formatted_date'; import { TimelineType } from '../../../../../common/types/timeline'; +const DescriptionCell = styled.span` + text-overflow: ellipsis; + display: -webkit-box; + -webkit-line-clamp: 5; + -webkit-box-orient: vertical; + overflow: hidden; +`; + /** * Returns the column definitions (passed as the `columns` prop to * `EuiBasicTable`) that are common to the compact `Open Timeline` modal view, @@ -85,9 +93,9 @@ export const getCommonColumns = ({ field: 'description', name: i18n.DESCRIPTION, render: (description: string) => ( - + {description != null && description.trim().length > 0 ? description : getEmptyTagValue()} - + ), sortable: false, }, From 4180a026b736058727466915801c109aa58fdd10 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 16 Jun 2021 10:22:26 +0200 Subject: [PATCH 08/14] [Lens] Formula overall functions (#99461) --- ...ns-public.expressionfunctiondefinitions.md | 1 + ...ssionfunctiondefinitions.overall_metric.md | 11 + ...ns-server.expressionfunctiondefinitions.md | 1 + ...ssionfunctiondefinitions.overall_metric.md | 11 + .../expression_functions/specs/index.ts | 1 + .../specs/overall_metric.ts | 168 +++++++ .../specs/tests/overall_metric.test.ts | 450 ++++++++++++++++++ .../common/expression_functions/types.ts | 2 + .../common/service/expressions_services.ts | 2 + src/plugins/expressions/public/public.api.md | 4 + src/plugins/expressions/server/server.api.md | 4 + .../dimension_panel/dimension_editor.tsx | 5 + .../definitions/calculations/index.ts | 10 + .../calculations/overall_metric.tsx | 224 +++++++++ .../definitions/calculations/utils.ts | 32 ++ .../operations/definitions/index.ts | 24 + .../operations/index.ts | 4 + .../operations/operations.test.ts | 16 + 18 files changed, 970 insertions(+) create mode 100644 docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md create mode 100644 docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md create mode 100644 src/plugins/expressions/common/expression_functions/specs/overall_metric.ts create mode 100644 src/plugins/expressions/common/expression_functions/specs/tests/overall_metric.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/overall_metric.tsx diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md index c6e00842a31e6a..2c03db82ba683a 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md @@ -21,6 +21,7 @@ export interface ExpressionFunctionDefinitions | [derivative](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.derivative.md) | ExpressionFunctionDerivative | | | [font](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [moving\_average](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.moving_average.md) | ExpressionFunctionMovingAverage | | +| [overall\_metric](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md) | ExpressionFunctionOverallMetric | | | [theme](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.theme.md) | ExpressionFunctionTheme | | | [var\_set](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.var_set.md) | ExpressionFunctionVarSet | | | [var](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.var.md) | ExpressionFunctionVar | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md new file mode 100644 index 00000000000000..8685788a2f3512 --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md) > [overall\_metric](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md) + +## ExpressionFunctionDefinitions.overall\_metric property + +Signature: + +```typescript +overall_metric: ExpressionFunctionOverallMetric; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md index 219678244951b4..f55fed99e1d3d4 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md @@ -21,6 +21,7 @@ export interface ExpressionFunctionDefinitions | [derivative](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.derivative.md) | ExpressionFunctionDerivative | | | [font](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.font.md) | ExpressionFunctionFont | | | [moving\_average](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.moving_average.md) | ExpressionFunctionMovingAverage | | +| [overall\_metric](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md) | ExpressionFunctionOverallMetric | | | [theme](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.theme.md) | ExpressionFunctionTheme | | | [var\_set](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.var_set.md) | ExpressionFunctionVarSet | | | [var](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.var.md) | ExpressionFunctionVar | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md new file mode 100644 index 00000000000000..b8564a696e6e48 --- /dev/null +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md) > [overall\_metric](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md) + +## ExpressionFunctionDefinitions.overall\_metric property + +Signature: + +```typescript +overall_metric: ExpressionFunctionOverallMetric; +``` diff --git a/src/plugins/expressions/common/expression_functions/specs/index.ts b/src/plugins/expressions/common/expression_functions/specs/index.ts index 20a6f9aac45674..c6d89f41d0e0d3 100644 --- a/src/plugins/expressions/common/expression_functions/specs/index.ts +++ b/src/plugins/expressions/common/expression_functions/specs/index.ts @@ -12,6 +12,7 @@ export * from './var_set'; export * from './var'; export * from './theme'; export * from './cumulative_sum'; +export * from './overall_metric'; export * from './derivative'; export * from './moving_average'; export * from './ui_setting'; diff --git a/src/plugins/expressions/common/expression_functions/specs/overall_metric.ts b/src/plugins/expressions/common/expression_functions/specs/overall_metric.ts new file mode 100644 index 00000000000000..e42112d3a23ed9 --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/overall_metric.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '../types'; +import { Datatable } from '../../expression_types'; +import { buildResultColumns, getBucketIdentifier } from '../series_calculation_helpers'; + +export interface OverallMetricArgs { + by?: string[]; + inputColumnId: string; + outputColumnId: string; + outputColumnName?: string; + metric: 'sum' | 'min' | 'max' | 'average'; +} + +export type ExpressionFunctionOverallMetric = ExpressionFunctionDefinition< + 'overall_metric', + Datatable, + OverallMetricArgs, + Datatable +>; + +function getValueAsNumberArray(value: unknown) { + if (Array.isArray(value)) { + return value.map((innerVal) => Number(innerVal)); + } else { + return [Number(value)]; + } +} + +/** + * Calculates the overall metric of a specified column in the data table. + * + * Also supports multiple series in a single data table - use the `by` argument + * to specify the columns to split the calculation by. + * For each unique combination of all `by` columns a separate overall metric will be calculated. + * The order of rows won't be changed - this function is not modifying any existing columns, it's only + * adding the specified `outputColumnId` column to every row of the table without adding or removing rows. + * + * Behavior: + * * Will write the overall metric of `inputColumnId` into `outputColumnId` + * * If provided will use `outputColumnName` as name for the newly created column. Otherwise falls back to `outputColumnId` + * * Each cell will contain the calculated metric based on the values of all cells belonging to the current series. + * + * Edge cases: + * * Will return the input table if `inputColumnId` does not exist + * * Will throw an error if `outputColumnId` exists already in provided data table + * * If the row value contains `null` or `undefined`, it will be ignored and overwritten with the overall metric of + * all cells of the same series. + * * For all values besides `null` and `undefined`, the value will be cast to a number before it's added to the + * overall metric of the current series - if this results in `NaN` (like in case of objects), all cells of the + * current series will be set to `NaN`. + * * To determine separate series defined by the `by` columns, the values of these columns will be cast to strings + * before comparison. If the values are objects, the return value of their `toString` method will be used for comparison. + * Missing values (`null` and `undefined`) will be treated as empty strings. + */ +export const overallMetric: ExpressionFunctionOverallMetric = { + name: 'overall_metric', + type: 'datatable', + + inputTypes: ['datatable'], + + help: i18n.translate('expressions.functions.overallMetric.help', { + defaultMessage: 'Calculates the overall sum, min, max or average of a column in a data table', + }), + + args: { + by: { + help: i18n.translate('expressions.functions.overallMetric.args.byHelpText', { + defaultMessage: 'Column to split the overall calculation by', + }), + multi: true, + types: ['string'], + required: false, + }, + metric: { + help: i18n.translate('expressions.functions.overallMetric.metricHelpText', { + defaultMessage: 'Metric to calculate', + }), + types: ['string'], + options: ['sum', 'min', 'max', 'average'], + }, + inputColumnId: { + help: i18n.translate('expressions.functions.overallMetric.args.inputColumnIdHelpText', { + defaultMessage: 'Column to calculate the overall metric of', + }), + types: ['string'], + required: true, + }, + outputColumnId: { + help: i18n.translate('expressions.functions.overallMetric.args.outputColumnIdHelpText', { + defaultMessage: 'Column to store the resulting overall metric in', + }), + types: ['string'], + required: true, + }, + outputColumnName: { + help: i18n.translate('expressions.functions.overallMetric.args.outputColumnNameHelpText', { + defaultMessage: 'Name of the column to store the resulting overall metric in', + }), + types: ['string'], + required: false, + }, + }, + + fn(input, { by, inputColumnId, outputColumnId, outputColumnName, metric }) { + const resultColumns = buildResultColumns( + input, + outputColumnId, + inputColumnId, + outputColumnName + ); + + if (!resultColumns) { + return input; + } + + const accumulators: Partial> = {}; + const valueCounter: Partial> = {}; + input.rows.forEach((row) => { + const bucketIdentifier = getBucketIdentifier(row, by); + const accumulatorValue = accumulators[bucketIdentifier] ?? 0; + + const currentValue = row[inputColumnId]; + if (currentValue != null) { + const currentNumberValues = getValueAsNumberArray(currentValue); + switch (metric) { + case 'average': + valueCounter[bucketIdentifier] = + (valueCounter[bucketIdentifier] ?? 0) + currentNumberValues.length; + case 'sum': + accumulators[bucketIdentifier] = + accumulatorValue + currentNumberValues.reduce((a, b) => a + b, 0); + break; + case 'min': + accumulators[bucketIdentifier] = Math.min(accumulatorValue, ...currentNumberValues); + break; + case 'max': + accumulators[bucketIdentifier] = Math.max(accumulatorValue, ...currentNumberValues); + break; + } + } + }); + if (metric === 'average') { + Object.keys(accumulators).forEach((bucketIdentifier) => { + accumulators[bucketIdentifier] = + accumulators[bucketIdentifier]! / valueCounter[bucketIdentifier]!; + }); + } + return { + ...input, + columns: resultColumns, + rows: input.rows.map((row) => { + const newRow = { ...row }; + const bucketIdentifier = getBucketIdentifier(row, by); + newRow[outputColumnId] = accumulators[bucketIdentifier]; + + return newRow; + }), + }; + }, +}; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/overall_metric.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/overall_metric.test.ts new file mode 100644 index 00000000000000..30354c4e54dc76 --- /dev/null +++ b/src/plugins/expressions/common/expression_functions/specs/tests/overall_metric.test.ts @@ -0,0 +1,450 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { functionWrapper } from './utils'; +import { ExecutionContext } from '../../../execution/types'; +import { Datatable } from '../../../expression_types/specs/datatable'; +import { overallMetric, OverallMetricArgs } from '../overall_metric'; + +describe('interpreter/functions#overall_metric', () => { + const fn = functionWrapper(overallMetric); + const runFn = (input: Datatable, args: OverallMetricArgs) => + fn(input, args, {} as ExecutionContext) as Datatable; + + it('calculates overall sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: 7 }, { val: 3 }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'sum' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([17, 17, 17, 17]); + }); + + it('ignores null or undefined', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{}, { val: null }, { val: undefined }, { val: 1 }, { val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'average' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([3, 3, 3, 3, 3]); + }); + + it('calculates overall sum for multiple series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3, split: 'B' }, + { val: 4, split: 'A' }, + { val: 5, split: 'A' }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'], metric: 'sum' } + ); + + expect(result.rows.map((row) => row.output)).toEqual([ + 1 + 4 + 5 + 6, + 2 + 3 + 7 + 8, + 2 + 3 + 7 + 8, + 1 + 4 + 5 + 6, + 1 + 4 + 5 + 6, + 1 + 4 + 5 + 6, + 2 + 3 + 7 + 8, + 2 + 3 + 7 + 8, + ]); + }); + + it('treats missing split column as separate series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'], metric: 'sum' } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + 1 + 4 + 6, + 2 + 7 + 8, + 3 + 5, + 1 + 4 + 6, + 3 + 5, + 1 + 4 + 6, + 2 + 7 + 8, + 2 + 7 + 8, + ]); + }); + + it('treats null like undefined and empty string for split columns', () => { + const table: Datatable = { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: null }, + { val: 8, split: 'B' }, + { val: 9, split: '' }, + ], + }; + + const result = runFn(table, { + inputColumnId: 'val', + outputColumnId: 'output', + by: ['split'], + metric: 'sum', + }); + expect(result.rows.map((row) => row.output)).toEqual([ + 1 + 4 + 6, + 2 + 8, + 3 + 5 + 7 + 9, + 1 + 4 + 6, + 3 + 5 + 7 + 9, + 1 + 4 + 6, + 3 + 5 + 7 + 9, + 2 + 8, + 3 + 5 + 7 + 9, + ]); + + const result2 = runFn(table, { + inputColumnId: 'val', + outputColumnId: 'output', + by: ['split'], + metric: 'max', + }); + expect(result2.rows.map((row) => row.output)).toEqual([6, 8, 9, 6, 9, 6, 9, 8, 9]); + }); + + it('handles array values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: [7, 10] }, { val: [3, 1] }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'sum' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([28, 28, 28, 28]); + }); + + it('takes array values into account for average calculation', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: [3, 4] }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'average' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([3, 3]); + }); + + it('handles array values for split columns', () => { + const table: Datatable = { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: [2, 11], split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: null }, + { val: 8, split: 'B' }, + { val: [9, 99], split: '' }, + ], + }; + + const result = runFn(table, { + inputColumnId: 'val', + outputColumnId: 'output', + by: ['split'], + metric: 'sum', + }); + expect(result.rows.map((row) => row.output)).toEqual([ + 1 + 4 + 6, + 2 + 11 + 8, + 3 + 5 + 7 + 9 + 99, + 1 + 4 + 6, + 3 + 5 + 7 + 9 + 99, + 1 + 4 + 6, + 3 + 5 + 7 + 9 + 99, + 2 + 11 + 8, + 3 + 5 + 7 + 9 + 99, + ]); + + const result2 = runFn(table, { + inputColumnId: 'val', + outputColumnId: 'output', + by: ['split'], + metric: 'max', + }); + expect(result2.rows.map((row) => row.output)).toEqual([6, 11, 99, 6, 99, 6, 99, 11, 99]); + }); + + it('calculates cumulative sum for multiple series by multiple split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + { id: 'split2', name: 'split2', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A', split2: 'C' }, + { val: 2, split: 'B', split2: 'C' }, + { val: 3, split2: 'C' }, + { val: 4, split: 'A', split2: 'C' }, + { val: 5 }, + { val: 6, split: 'A', split2: 'D' }, + { val: 7, split: 'B', split2: 'D' }, + { val: 8, split: 'B', split2: 'D' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split', 'split2'], metric: 'sum' } + ); + expect(result.rows.map((row) => row.output)).toEqual([1 + 4, 2, 3, 1 + 4, 5, 6, 7 + 8, 7 + 8]); + }); + + it('splits separate series by the string representation of the cell values', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: { anObj: 3 } }, + { val: 2, split: { anotherObj: 5 } }, + { val: 10, split: 5 }, + { val: 11, split: '5' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'], metric: 'sum' } + ); + + expect(result.rows.map((row) => row.output)).toEqual([1 + 2, 1 + 2, 10 + 11, 10 + 11]); + }); + + it('casts values to number before calculating cumulative sum', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: '3' }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'max' } + ); + expect(result.rows.map((row) => row.output)).toEqual([7, 7, 7, 7]); + }); + + it('casts values to number before calculating metric for NaN like values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: {} }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'min' } + ); + expect(result.rows.map((row) => row.output)).toEqual([NaN, NaN, NaN, NaN]); + }); + + it('skips undefined and null values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [ + { val: null }, + { val: 7 }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: undefined }, + { val: '3' }, + { val: 2 }, + { val: null }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'average' } + ); + expect(result.rows.map((row) => row.output)).toEqual([4, 4, 4, 4, 4, 4, 4, 4, 4]); + }); + + it('copies over meta information from the source column', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', metric: 'sum' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }); + }); + + it('sets output name on output column if specified', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { + inputColumnId: 'val', + outputColumnId: 'output', + outputColumnName: 'Output name', + metric: 'min', + } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'Output name', + meta: { type: 'number' }, + }); + }); + + it('returns source table if input column does not exist', () => { + const input: Datatable = { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }; + expect( + runFn(input, { inputColumnId: 'nonexisting', outputColumnId: 'output', metric: 'sum' }) + ).toBe(input); + }); + + it('throws an error if output column exists already', () => { + expect(() => + runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'val', metric: 'max' } + ) + ).toThrow(); + }); +}); diff --git a/src/plugins/expressions/common/expression_functions/types.ts b/src/plugins/expressions/common/expression_functions/types.ts index e1378a27bdfc29..0ec61b39608a05 100644 --- a/src/plugins/expressions/common/expression_functions/types.ts +++ b/src/plugins/expressions/common/expression_functions/types.ts @@ -18,6 +18,7 @@ import { ExpressionFunctionCumulativeSum, ExpressionFunctionDerivative, ExpressionFunctionMovingAverage, + ExpressionFunctionOverallMetric, } from './specs'; import { ExpressionAstFunction } from '../ast'; import { PersistableStateDefinition } from '../../../kibana_utils/common'; @@ -119,6 +120,7 @@ export interface ExpressionFunctionDefinitions { var: ExpressionFunctionVar; theme: ExpressionFunctionTheme; cumulative_sum: ExpressionFunctionCumulativeSum; + overall_metric: ExpressionFunctionOverallMetric; derivative: ExpressionFunctionDerivative; moving_average: ExpressionFunctionMovingAverage; } diff --git a/src/plugins/expressions/common/service/expressions_services.ts b/src/plugins/expressions/common/service/expressions_services.ts index a8839c9b0d71e1..f7afc12aa96bad 100644 --- a/src/plugins/expressions/common/service/expressions_services.ts +++ b/src/plugins/expressions/common/service/expressions_services.ts @@ -29,6 +29,7 @@ import { derivative, movingAverage, mapColumn, + overallMetric, math, } from '../expression_functions'; @@ -340,6 +341,7 @@ export class ExpressionsService implements PersistableStateService { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) + .filter( + (operationDefinition) => + !('selectionStyle' in operationDefinition) || + operationDefinition.selectionStyle !== 'hidden' + ) .filter(({ type }) => fieldByOperation[type]?.size || operationWithoutField.has(type)) .sort((op1, op2) => { return op1.displayName.localeCompare(op2.displayName); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts index 815acb8c4169fb..a7741bc60d6469 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts @@ -9,3 +9,13 @@ export { counterRateOperation, CounterRateIndexPatternColumn } from './counter_r export { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum'; export { derivativeOperation, DerivativeIndexPatternColumn } from './differences'; export { movingAverageOperation, MovingAverageIndexPatternColumn } from './moving_average'; +export { + overallSumOperation, + OverallSumIndexPatternColumn, + overallMinOperation, + OverallMinIndexPatternColumn, + overallMaxOperation, + OverallMaxIndexPatternColumn, + overallAverageOperation, + OverallAverageIndexPatternColumn, +} from './overall_metric'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/overall_metric.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/overall_metric.tsx new file mode 100644 index 00000000000000..21ec5387b38539 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/overall_metric.tsx @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { optionallHistogramBasedOperationToExpression } from './utils'; +import { OperationDefinition } from '..'; +import { getFormatFromPreviousColumn } from '../helpers'; + +type OverallMetricIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: T; + }; + +export type OverallSumIndexPatternColumn = OverallMetricIndexPatternColumn<'overall_sum'>; +export type OverallMinIndexPatternColumn = OverallMetricIndexPatternColumn<'overall_min'>; +export type OverallMaxIndexPatternColumn = OverallMetricIndexPatternColumn<'overall_max'>; +export type OverallAverageIndexPatternColumn = OverallMetricIndexPatternColumn<'overall_average'>; + +function buildOverallMetricOperation>({ + type, + displayName, + ofName, + description, + metric, +}: { + type: T['operationType']; + displayName: string; + ofName: (name?: string) => string; + description: string; + metric: string; +}): OperationDefinition { + return { + type, + priority: 1, + displayName, + input: 'fullReference', + selectionStyle: 'hidden', + requiredReferences: [ + { + input: ['field', 'managedReference', 'fullReference'], + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: (column, indexPattern, columns) => { + const ref = columns[column.references[0]]; + return ofName( + ref && 'sourceField' in ref + ? indexPattern.getFieldByName(ref.sourceField)?.displayName + : undefined + ); + }, + toExpression: (layer, columnId) => { + return optionallHistogramBasedOperationToExpression(layer, columnId, 'overall_metric', { + metric: [metric], + }); + }, + buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => { + const ref = layer.columns[referenceIds[0]]; + return { + label: ofName( + ref && 'sourceField' in ref + ? indexPattern.getFieldByName(ref.sourceField)?.displayName + : undefined + ), + dataType: 'number', + operationType: 'overall_sum', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: getFormatFromPreviousColumn(previousColumn), + } as T; + }, + isTransferable: () => { + return true; + }, + filterable: false, + shiftable: false, + documentation: { + section: 'calculation', + signature: i18n.translate('xpack.lens.indexPattern.overall_metric', { + defaultMessage: 'metric: number', + }), + description, + }, + }; +} + +export const overallSumOperation = buildOverallMetricOperation({ + type: 'overall_sum', + displayName: i18n.translate('xpack.lens.indexPattern.overallSum', { + defaultMessage: 'Overall sum', + }), + ofName: (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.overallSumOf', { + defaultMessage: 'Overall sum of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); + }, + metric: 'sum', + description: i18n.translate('xpack.lens.indexPattern.overall_sum.documentation', { + defaultMessage: ` +Calculates the sum of a metric of all data points of a series in the current chart. A series is defined by a dimension using a date histogram or interval function. +Other dimensions breaking down the data like top values or filter are treated as separate series. + +If no date histograms or interval functions are used in the current chart, \`overall_sum\` is calculating the sum over all dimensions no matter the used function. + +Example: Percentage of total +\`sum(bytes) / overall_sum(sum(bytes))\` + `, + }), +}); + +export const overallMinOperation = buildOverallMetricOperation({ + type: 'overall_min', + displayName: i18n.translate('xpack.lens.indexPattern.overallMin', { + defaultMessage: 'Overall min', + }), + ofName: (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.overallMinOf', { + defaultMessage: 'Overall min of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); + }, + metric: 'min', + description: i18n.translate('xpack.lens.indexPattern.overall_min.documentation', { + defaultMessage: ` +Calculates the minimum of a metric for all data points of a series in the current chart. A series is defined by a dimension using a date histogram or interval function. +Other dimensions breaking down the data like top values or filter are treated as separate series. + +If no date histograms or interval functions are used in the current chart, \`overall_min\` is calculating the minimum over all dimensions no matter the used function + +Example: Percentage of range +\`(sum(bytes) - overall_min(sum(bytes)) / (overall_max(bytes) - overall_min(bytes))\` + `, + }), +}); + +export const overallMaxOperation = buildOverallMetricOperation({ + type: 'overall_max', + displayName: i18n.translate('xpack.lens.indexPattern.overallMax', { + defaultMessage: 'Overall max', + }), + ofName: (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.overallMaxOf', { + defaultMessage: 'Overall max of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); + }, + metric: 'max', + description: i18n.translate('xpack.lens.indexPattern.overall_max.documentation', { + defaultMessage: ` +Calculates the maximum of a metric for all data points of a series in the current chart. A series is defined by a dimension using a date histogram or interval function. +Other dimensions breaking down the data like top values or filter are treated as separate series. + +If no date histograms or interval functions are used in the current chart, \`overall_max\` is calculating the maximum over all dimensions no matter the used function + +Example: Percentage of range +\`(sum(bytes) - overall_min(sum(bytes)) / (overall_max(bytes) - overall_min(bytes))\` + `, + }), +}); + +export const overallAverageOperation = buildOverallMetricOperation( + { + type: 'overall_average', + displayName: i18n.translate('xpack.lens.indexPattern.overallMax', { + defaultMessage: 'Overall max', + }), + ofName: (name?: string) => { + return i18n.translate('xpack.lens.indexPattern.overallAverageOf', { + defaultMessage: 'Overall average of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, + }); + }, + metric: 'average', + description: i18n.translate('xpack.lens.indexPattern.overall_average.documentation', { + defaultMessage: ` +Calculates the average of a metric for all data points of a series in the current chart. A series is defined by a dimension using a date histogram or interval function. +Other dimensions breaking down the data like top values or filter are treated as separate series. + +If no date histograms or interval functions are used in the current chart, \`overall_average\` is calculating the average over all dimensions no matter the used function + +Example: Divergence from the mean: +\`sum(bytes) - overall_average(sum(bytes))\` + `, + }), + } +); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts index 1f4f097c6a7fb5..03b9d6c07709c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -134,3 +134,35 @@ export function dateBasedOperationToExpression( }, ]; } + +/** + * Creates an expression ast for a date based operation (cumulative sum, derivative, moving average, counter rate) + */ +export function optionallHistogramBasedOperationToExpression( + layer: IndexPatternLayer, + columnId: string, + functionName: string, + additionalArgs: Record = {} +): ExpressionFunctionAST[] { + const currentColumn = (layer.columns[columnId] as unknown) as ReferenceBasedIndexPatternColumn; + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const nonHistogramColumns = buckets.filter( + (colId) => + layer.columns[colId].operationType !== 'date_histogram' && + layer.columns[colId].operationType !== 'range' + )!; + + return [ + { + type: 'function', + function: functionName, + arguments: { + by: nonHistogramColumns.length === buckets.length ? [] : nonHistogramColumns, + inputColumnId: [currentColumn.references[0]], + outputColumnId: [columnId], + outputColumnName: [currentColumn.label], + ...additionalArgs, + }, + }, + ]; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index a7bf415817797d..c38475f85f47e5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -33,6 +33,14 @@ import { DerivativeIndexPatternColumn, movingAverageOperation, MovingAverageIndexPatternColumn, + OverallSumIndexPatternColumn, + overallSumOperation, + OverallMinIndexPatternColumn, + overallMinOperation, + OverallMaxIndexPatternColumn, + overallMaxOperation, + OverallAverageIndexPatternColumn, + overallAverageOperation, } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; import { @@ -71,6 +79,10 @@ export type IndexPatternColumn = | CountIndexPatternColumn | LastValueIndexPatternColumn | CumulativeSumIndexPatternColumn + | OverallSumIndexPatternColumn + | OverallMinIndexPatternColumn + | OverallMaxIndexPatternColumn + | OverallAverageIndexPatternColumn | CounterRateIndexPatternColumn | DerivativeIndexPatternColumn | MovingAverageIndexPatternColumn @@ -98,6 +110,10 @@ export { CounterRateIndexPatternColumn, DerivativeIndexPatternColumn, MovingAverageIndexPatternColumn, + OverallSumIndexPatternColumn, + OverallMinIndexPatternColumn, + OverallMaxIndexPatternColumn, + OverallAverageIndexPatternColumn, } from './calculations'; export { CountIndexPatternColumn } from './count'; export { LastValueIndexPatternColumn } from './last_value'; @@ -126,6 +142,10 @@ const internalOperationDefinitions = [ movingAverageOperation, mathOperation, formulaOperation, + overallSumOperation, + overallMinOperation, + overallMaxOperation, + overallAverageOperation, ]; export { termsOperation } from './terms'; @@ -141,6 +161,10 @@ export { counterRateOperation, derivativeOperation, movingAverageOperation, + overallSumOperation, + overallAverageOperation, + overallMaxOperation, + overallMinOperation, } from './calculations'; export { formulaOperation } from './formula/formula'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts index aa46dd765bd8b8..d55c5d3c00f179 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts @@ -31,4 +31,8 @@ export { CounterRateIndexPatternColumn, DerivativeIndexPatternColumn, MovingAverageIndexPatternColumn, + OverallSumIndexPatternColumn, + OverallMinIndexPatternColumn, + OverallMaxIndexPatternColumn, + OverallAverageIndexPatternColumn, } from './definitions'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 7df096c27d9a0d..2ed6e2b3a7bcb0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -319,6 +319,22 @@ describe('getOperationTypesForField', () => { "operationType": "moving_average", "type": "fullReference", }, + Object { + "operationType": "overall_sum", + "type": "fullReference", + }, + Object { + "operationType": "overall_min", + "type": "fullReference", + }, + Object { + "operationType": "overall_max", + "type": "fullReference", + }, + Object { + "operationType": "overall_average", + "type": "fullReference", + }, Object { "field": "bytes", "operationType": "min", From a946f9bd3da37e44da480e61dd963bcf8b9f2a80 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 16 Jun 2021 10:23:31 +0200 Subject: [PATCH 09/14] add description and owner to kibana.json (#102238) --- x-pack/plugins/lens/kibana.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index a5c19911f60b94..c5b3e72f39159d 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -36,5 +36,10 @@ "kibanaUtils", "kibanaReact", "embeddable" - ] + ], + "owner": { + "name": "Kibana App", + "githubTeam": "kibana-app" + }, + "description": "Visualization editor allowing to quickly and easily configure compelling visualizations to use on dashboards and canvas workpads. Exposes components to embed visualizations and link into the Lens editor from within other apps in Kibana." } From aea2a384fd49e187dfc5edcf365faa0726d0b7ce Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 16 Jun 2021 10:24:13 +0200 Subject: [PATCH 10/14] Do not send other bucket request on exhaustive terms list (#102097) --- .../_terms_other_bucket_helper.test.ts | 62 +++++++++++++++++++ .../buckets/_terms_other_bucket_helper.ts | 6 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 2aa0d346afe343..523bbe1f010181 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -174,6 +174,57 @@ const nestedTermResponse = { status: 200, }; +const exhaustiveNestedTermResponse = { + took: 10, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 14005, + max_score: 0, + hits: [], + }, + aggregations: { + '1': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 8325, + buckets: [ + { + '2': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { key: 'ios', doc_count: 2850 }, + { key: 'win xp', doc_count: 2830 }, + { key: '__missing__', doc_count: 1430 }, + ], + }, + key: 'US-with-dash', + doc_count: 2850, + }, + { + '2': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { key: 'ios', doc_count: 1850 }, + { key: 'win xp', doc_count: 1830 }, + { key: '__missing__', doc_count: 130 }, + ], + }, + key: 'IN-with-dash', + doc_count: 2830, + }, + ], + }, + }, + status: 200, +}; + const nestedTermResponseNoResults = { took: 10, timed_out: false, @@ -326,6 +377,17 @@ describe('Terms Agg Other bucket helper', () => { } }); + test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + expect( + buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + exhaustiveNestedTermResponse + ) + ).toBeFalsy(); + }); + test('excludes exists filter for scripted fields', () => { const aggConfigs = getAggConfigs(nestedTerm.aggs); aggConfigs.aggs[1].params.field.scripted = true; diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 372d487bcf7a39..2a1cd873f62822 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -156,6 +156,7 @@ export const buildOtherBucketAgg = ( }; let noAggBucketResults = false; + let exhaustiveBuckets = true; // recursively create filters for all parent aggregation buckets const walkBucketTree = ( @@ -175,6 +176,9 @@ export const buildOtherBucketAgg = ( const newAggIndex = aggIndex + 1; const newAgg = bucketAggs[newAggIndex]; const currentAgg = bucketAggs[aggIndex]; + if (aggIndex === index && agg && agg.sum_other_doc_count > 0) { + exhaustiveBuckets = false; + } if (aggIndex < index) { each(agg.buckets, (bucket: any, bucketObjKey) => { const bucketKey = currentAgg.getKey( @@ -223,7 +227,7 @@ export const buildOtherBucketAgg = ( walkBucketTree(0, response.aggregations, bucketAggs[0].id, [], ''); // bail if there were no bucket results - if (noAggBucketResults) { + if (noAggBucketResults || exhaustiveBuckets) { return false; } From 5c90bacce543f4549b57252f7efa8ea0925c09a5 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Wed, 16 Jun 2021 10:55:22 +0200 Subject: [PATCH 11/14] do not throw execa error when building ts refs (#102154) --- src/dev/typescript/build_ts_refs.ts | 16 ++++++++++++---- src/dev/typescript/run_type_check_cli.ts | 6 +++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/dev/typescript/build_ts_refs.ts b/src/dev/typescript/build_ts_refs.ts index 2e25827996e453..26425b7a3e61df 100644 --- a/src/dev/typescript/build_ts_refs.ts +++ b/src/dev/typescript/build_ts_refs.ts @@ -13,12 +13,20 @@ import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils'; export const REF_CONFIG_PATHS = [Path.resolve(REPO_ROOT, 'tsconfig.refs.json')]; -export async function buildAllTsRefs(log: ToolingLog) { +export async function buildAllTsRefs(log: ToolingLog): Promise<{ failed: boolean }> { for (const path of REF_CONFIG_PATHS) { const relative = Path.relative(REPO_ROOT, path); log.debug(`Building TypeScript projects refs for ${relative}...`); - await execa(require.resolve('typescript/bin/tsc'), ['-b', relative, '--pretty'], { - cwd: REPO_ROOT, - }); + const { failed, stdout } = await execa( + require.resolve('typescript/bin/tsc'), + ['-b', relative, '--pretty'], + { + cwd: REPO_ROOT, + reject: false, + } + ); + log.info(stdout); + if (failed) return { failed }; } + return { failed: false }; } diff --git a/src/dev/typescript/run_type_check_cli.ts b/src/dev/typescript/run_type_check_cli.ts index f95c230f44b9e4..d9e9eb036fe0f2 100644 --- a/src/dev/typescript/run_type_check_cli.ts +++ b/src/dev/typescript/run_type_check_cli.ts @@ -69,7 +69,11 @@ export async function runTypeCheckCli() { process.exit(); } - await buildAllTsRefs(log); + const { failed } = await buildAllTsRefs(log); + if (failed) { + log.error('Unable to build TS project refs'); + process.exit(1); + } const tscArgs = [ // composite project cannot be used with --noEmit From aa97040bb6acb32b67b2a4c005ca905c50c5a5c9 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 16 Jun 2021 10:01:43 +0100 Subject: [PATCH 12/14] [APM-UI][e2e] discard CI builds more often (#102217) --- .ci/end2end.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/end2end.groovy b/.ci/end2end.groovy index 87b64437deafcd..f1095f8035b6c4 100644 --- a/.ci/end2end.groovy +++ b/.ci/end2end.groovy @@ -13,12 +13,12 @@ pipeline { BASE_DIR = 'src/github.com/elastic/kibana' HOME = "${env.WORKSPACE}" E2E_DIR = 'x-pack/plugins/apm/e2e' - PIPELINE_LOG_LEVEL = 'DEBUG' + PIPELINE_LOG_LEVEL = 'INFO' KBN_OPTIMIZER_THEMES = 'v7light' } options { timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '40', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '10', daysToKeepStr: '30')) timestamps() ansiColor('xterm') disableResume() From 037d7aeb8f11057e0b4696e9f6aeafd321b32220 Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Wed, 16 Jun 2021 12:23:10 +0200 Subject: [PATCH 13/14] Enhance cases bulk deletion action dialog message (#101403) Differentiate the dialog message on the deletion of one item from the deletion of multiple items. Simplifies CasesTableUtilityBar by handling the selection of multiple and single cases in the same way. --- x-pack/plugins/cases/common/ui/types.ts | 2 +- .../cases/public/common/translations.ts | 12 +++---- .../public/components/all_cases/actions.tsx | 4 +-- .../public/components/all_cases/columns.tsx | 1 - .../components/all_cases/utility_bar.tsx | 33 +++++-------------- .../components/case_action_bar/actions.tsx | 3 +- .../components/confirm_delete_case/index.tsx | 18 ++++------ .../confirm_delete_case/translations.ts | 26 +++++---------- .../containers/use_delete_cases.test.tsx | 6 ++-- 9 files changed, 36 insertions(+), 69 deletions(-) diff --git a/x-pack/plugins/cases/common/ui/types.ts b/x-pack/plugins/cases/common/ui/types.ts index 284f5e706292cc..1dbb633e32adf0 100644 --- a/x-pack/plugins/cases/common/ui/types.ts +++ b/x-pack/plugins/cases/common/ui/types.ts @@ -153,7 +153,7 @@ export interface ActionLicense { export interface DeleteCase { id: string; type: CaseType | null; - title?: string; + title: string; } export interface FieldMappings { diff --git a/x-pack/plugins/cases/public/common/translations.ts b/x-pack/plugins/cases/public/common/translations.ts index 85cfb60b1d6b81..f1bfde4cc44851 100644 --- a/x-pack/plugins/cases/public/common/translations.ts +++ b/x-pack/plugins/cases/public/common/translations.ts @@ -30,13 +30,11 @@ export const CANCEL = i18n.translate('xpack.cases.caseView.cancel', { defaultMessage: 'Cancel', }); -export const DELETE_CASE = i18n.translate('xpack.cases.confirmDeleteCase.deleteCase', { - defaultMessage: 'Delete case', -}); - -export const DELETE_CASES = i18n.translate('xpack.cases.confirmDeleteCase.deleteCases', { - defaultMessage: 'Delete cases', -}); +export const DELETE_CASE = (quantity: number = 1) => + i18n.translate('xpack.cases.confirmDeleteCase.deleteCase', { + values: { quantity }, + defaultMessage: `Delete {quantity, plural, =1 {case} other {cases}}`, + }); export const NAME = i18n.translate('xpack.cases.caseView.name', { defaultMessage: 'Name', diff --git a/x-pack/plugins/cases/public/components/all_cases/actions.tsx b/x-pack/plugins/cases/public/components/all_cases/actions.tsx index 8742b8fea23a42..4820b10308934f 100644 --- a/x-pack/plugins/cases/public/components/all_cases/actions.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/actions.tsx @@ -80,9 +80,9 @@ export const getActions = ({ makeInProgressAction, closeCaseAction, { - description: i18n.DELETE_CASE, + description: i18n.DELETE_CASE(), icon: 'trash', - name: i18n.DELETE_CASE, + name: i18n.DELETE_CASE(), onClick: deleteCaseOnClick, type: 'icon', 'data-test-subj': 'action-delete', diff --git a/x-pack/plugins/cases/public/components/all_cases/columns.tsx b/x-pack/plugins/cases/public/components/all_cases/columns.tsx index 947d405d188cf0..a5a299851d975a 100644 --- a/x-pack/plugins/cases/public/components/all_cases/columns.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/columns.tsx @@ -306,7 +306,6 @@ export const useCasesColumns = ({ diff --git a/x-pack/plugins/cases/public/components/all_cases/utility_bar.tsx b/x-pack/plugins/cases/public/components/all_cases/utility_bar.tsx index d0981c38385e96..a2b4c14c0278a8 100644 --- a/x-pack/plugins/cases/public/components/all_cases/utility_bar.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/utility_bar.tsx @@ -41,12 +41,8 @@ export const CasesTableUtilityBar: FunctionComponent = ({ refreshCases, selectedCases, }) => { - const [deleteBulk, setDeleteBulk] = useState([]); - const [deleteThisCase, setDeleteThisCase] = useState({ - title: '', - id: '', - type: null, - }); + const [deleteCases, setDeleteCases] = useState([]); + // Delete case const { dispatchResetIsDeleted, @@ -86,24 +82,15 @@ export const CasesTableUtilityBar: FunctionComponent = ({ const toggleBulkDeleteModal = useCallback( (cases: Case[]) => { handleToggleModal(); - if (cases.length === 1) { - const singleCase = cases[0]; - if (singleCase) { - return setDeleteThisCase({ - id: singleCase.id, - title: singleCase.title, - type: singleCase.type, - }); - } - } + const convertToDeleteCases: DeleteCase[] = cases.map(({ id, title, type }) => ({ id, title, type, })); - setDeleteBulk(convertToDeleteCases); + setDeleteCases(convertToDeleteCases); }, - [setDeleteBulk, handleToggleModal] + [setDeleteCases, handleToggleModal] ); const handleUpdateCaseStatus = useCallback( @@ -128,6 +115,7 @@ export const CasesTableUtilityBar: FunctionComponent = ({ ), [selectedCases, filterOptions.status, toggleBulkDeleteModal, handleUpdateCaseStatus] ); + return ( @@ -159,14 +147,11 @@ export const CasesTableUtilityBar: FunctionComponent = ({ 0} + caseQuantity={deleteCases.length} onCancel={handleToggleModal} - onConfirm={handleOnDeleteConfirm.bind( - null, - deleteBulk.length > 0 ? deleteBulk : [deleteThisCase] - )} + onConfirm={handleOnDeleteConfirm.bind(null, deleteCases)} /> ); diff --git a/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx b/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx index 922ffd09aaac9d..c2578dc3debdb6 100644 --- a/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx +++ b/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx @@ -41,7 +41,7 @@ const ActionsComponent: React.FC = ({ { disabled, iconType: 'trash', - label: i18n.DELETE_CASE, + label: i18n.DELETE_CASE(), onClick: handleToggleModal, }, ...(currentExternalIncident != null && !isEmpty(currentExternalIncident?.externalUrl) @@ -67,7 +67,6 @@ const ActionsComponent: React.FC = ({ void; onConfirm: () => void; } @@ -20,7 +20,7 @@ interface ConfirmDeleteCaseModalProps { const ConfirmDeleteCaseModalComp: React.FC = ({ caseTitle, isModalVisible, - isPlural, + caseQuantity = 1, onCancel, onConfirm, }) => { @@ -31,20 +31,14 @@ const ConfirmDeleteCaseModalComp: React.FC = ({ - {isPlural ? i18n.CONFIRM_QUESTION_PLURAL : i18n.CONFIRM_QUESTION} + {i18n.CONFIRM_QUESTION(caseQuantity)} ); }; diff --git a/x-pack/plugins/cases/public/components/confirm_delete_case/translations.ts b/x-pack/plugins/cases/public/components/confirm_delete_case/translations.ts index 0400c4c7fef413..f8e4ab2a83a738 100644 --- a/x-pack/plugins/cases/public/components/confirm_delete_case/translations.ts +++ b/x-pack/plugins/cases/public/components/confirm_delete_case/translations.ts @@ -14,23 +14,15 @@ export const DELETE_TITLE = (caseTitle: string) => defaultMessage: 'Delete "{caseTitle}"', }); -export const DELETE_THIS_CASE = (caseTitle: string) => - i18n.translate('xpack.cases.confirmDeleteCase.deleteThisCase', { - defaultMessage: 'Delete this case', +export const DELETE_SELECTED_CASES = (quantity: number, title: string) => + i18n.translate('xpack.cases.confirmDeleteCase.selectedCases', { + values: { quantity, title }, + defaultMessage: 'Delete "{quantity, plural, =1 {{title}} other {Selected {quantity} cases}}"', }); -export const CONFIRM_QUESTION = i18n.translate('xpack.cases.confirmDeleteCase.confirmQuestion', { - defaultMessage: - 'By deleting this case, all related case data will be permanently removed and you will no longer be able to push data to an external incident management system. Are you sure you wish to proceed?', -}); -export const DELETE_SELECTED_CASES = i18n.translate('xpack.cases.confirmDeleteCase.selectedCases', { - defaultMessage: 'Delete selected cases', -}); - -export const CONFIRM_QUESTION_PLURAL = i18n.translate( - 'xpack.cases.confirmDeleteCase.confirmQuestionPlural', - { +export const CONFIRM_QUESTION = (quantity: number) => + i18n.translate('xpack.cases.confirmDeleteCase.confirmQuestion', { + values: { quantity }, defaultMessage: - 'By deleting these cases, all related case data will be permanently removed and you will no longer be able to push data to an external incident management system. Are you sure you wish to proceed?', - } -); + 'By deleting {quantity, plural, =1 {this case} other {these cases}}, all related case data will be permanently removed and you will no longer be able to push data to an external incident management system. Are you sure you wish to proceed?', + }); diff --git a/x-pack/plugins/cases/public/containers/use_delete_cases.test.tsx b/x-pack/plugins/cases/public/containers/use_delete_cases.test.tsx index e86ed0c036974a..691af580b333a8 100644 --- a/x-pack/plugins/cases/public/containers/use_delete_cases.test.tsx +++ b/x-pack/plugins/cases/public/containers/use_delete_cases.test.tsx @@ -17,9 +17,9 @@ jest.mock('../common/lib/kibana'); describe('useDeleteCases', () => { const abortCtrl = new AbortController(); const deleteObj = [ - { id: '1', type: CaseType.individual }, - { id: '2', type: CaseType.individual }, - { id: '3', type: CaseType.individual }, + { id: '1', type: CaseType.individual, title: 'case 1' }, + { id: '2', type: CaseType.individual, title: 'case 2' }, + { id: '3', type: CaseType.individual, title: 'case 3' }, ]; const deleteArr = ['1', '2', '3']; it('init', async () => { From 35e10ba77013715e381a2373790d67d94df825f1 Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Wed, 16 Jun 2021 11:31:10 +0100 Subject: [PATCH 14/14] [ML] Adds optimizations for Logs UI anomaly detection jobs (#102191) * [ML] Adds optimizations for Logs UI anomaly detection jobs * [ML] Increment version for log-entry-categories-count job Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../containers/logs/log_analysis/api/ml_setup_module_api.ts | 4 ++++ .../modules/log_entry_categories/module_descriptor.ts | 1 + .../log_analysis/modules/log_entry_rate/module_descriptor.ts | 1 + .../logs_ui_categories/ml/log_entry_categories_count.json | 4 ++-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts index ea1567d6056f1d..6304471e818fa9 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts @@ -21,6 +21,7 @@ interface RequestArgs { jobOverrides?: SetupMlModuleJobOverrides[]; datafeedOverrides?: SetupMlModuleDatafeedOverrides[]; query?: object; + useDedicatedIndex?: boolean; } export const callSetupMlModuleAPI = async (requestArgs: RequestArgs, fetch: HttpHandler) => { @@ -34,6 +35,7 @@ export const callSetupMlModuleAPI = async (requestArgs: RequestArgs, fetch: Http jobOverrides = [], datafeedOverrides = [], query, + useDedicatedIndex = false, } = requestArgs; const response = await fetch(`/api/ml/modules/setup/${moduleId}`, { @@ -48,6 +50,7 @@ export const callSetupMlModuleAPI = async (requestArgs: RequestArgs, fetch: Http jobOverrides, datafeedOverrides, query, + useDedicatedIndex, }) ), }); @@ -78,6 +81,7 @@ const setupMlModuleRequestParamsRT = rt.intersection([ startDatafeed: rt.boolean, jobOverrides: rt.array(setupMlModuleJobOverridesRT), datafeedOverrides: rt.array(setupMlModuleDatafeedOverridesRT), + useDedicatedIndex: rt.boolean, }), rt.exact( rt.partial({ diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts index af2bd1802042a4..6823ed173a740c 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts @@ -124,6 +124,7 @@ const setUpModule = async ( jobOverrides, datafeedOverrides, query, + useDedicatedIndex: true, }, fetch ); diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts index 9704afd80e9ea6..c4c939d0ebb9d5 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts @@ -116,6 +116,7 @@ const setUpModule = async ( jobOverrides, datafeedOverrides, query, + useDedicatedIndex: true, }, fetch ); diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json index ad7da3330bb6cd..90f88275cb6d0b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json @@ -22,7 +22,7 @@ ], "per_partition_categorization": { "enabled": true, - "stop_on_warn": false + "stop_on_warn": true } }, "analysis_limits": { @@ -38,6 +38,6 @@ }, "custom_settings": { "created_by": "ml-module-logs-ui-categories", - "job_revision": 1 + "job_revision": 2 } }