[Transform] Support for the top_metrics aggregation (#101152)

* [ML] init top_metrics agg

* [ML] support sort

* [ML] support _score sorting

* [ML] support sort mode

* [ML] support numeric type sorting

* [ML] update field label, hide additional sorting controls

* [ML] preserve advanced config

* [ML] update agg fields after runtime fields edit

* [ML] fix TS issue with EuiButtonGroup

* [ML] fix Field label

* [ML] refactor setUiConfig

* [ML] update unit tests

* [ML] wrap advanced sorting settings with accordion

* [ML] config validation with tests

* [ML] fix preserving of the unsupported config

* [ML] update translation message

* [ML] fix level of the custom config

* [ML] preserve unsupported config for sorting

x-pack/plugins/transform/common/types/pivot_aggs.ts

@@ -17,6 +17,7 @@ export const PIVOT_SUPPORTED_AGGS = {
   SUM: 'sum',
   VALUE_COUNT: 'value_count',
   FILTER: 'filter',
+  TOP_METRICS: 'top_metrics',
 } as const;
 
 export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS];

x-pack/plugins/transform/public/app/common/pivot_aggs.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { getAggConfigFromEsAgg } from './pivot_aggs';
+import { getAggConfigFromEsAgg, isSpecialSortField } from './pivot_aggs';
 import {
   FilterAggForm,
   FilterTermForm,
@@ -67,3 +67,12 @@ describe('getAggConfigFromEsAgg', () => {
     });
   });
 });
+
+describe('isSpecialSortField', () => {
+  test('detects special sort field', () => {
+    expect(isSpecialSortField('_score')).toBe(true);
+  });
+  test('rejects special fields that not supported yet', () => {
+    expect(isSpecialSortField('_doc')).toBe(false);
+  });
+});

x-pack/plugins/transform/public/app/common/pivot_aggs.ts

@@ -7,7 +7,7 @@
 
 import { FC } from 'react';
 
-import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
+import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
 
 import type { AggName } from '../../../common/types/aggregations';
 import type { Dictionary } from '../../../common/types/common';
@@ -43,6 +43,7 @@ export const pivotAggsFieldSupport = {
     PIVOT_SUPPORTED_AGGS.CARDINALITY,
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
     PIVOT_SUPPORTED_AGGS.FILTER,
+    PIVOT_SUPPORTED_AGGS.TOP_METRICS,
   ],
   [KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
   [KBN_FIELD_TYPES.NUMBER]: [
@@ -54,17 +55,78 @@ export const pivotAggsFieldSupport = {
     PIVOT_SUPPORTED_AGGS.SUM,
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
     PIVOT_SUPPORTED_AGGS.FILTER,
+    PIVOT_SUPPORTED_AGGS.TOP_METRICS,
   ],
   [KBN_FIELD_TYPES.STRING]: [
     PIVOT_SUPPORTED_AGGS.CARDINALITY,
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
     PIVOT_SUPPORTED_AGGS.FILTER,
+    PIVOT_SUPPORTED_AGGS.TOP_METRICS,
   ],
   [KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
   [KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
   [KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
 };
 
+export const TOP_METRICS_SORT_FIELD_TYPES = [
+  KBN_FIELD_TYPES.NUMBER,
+  KBN_FIELD_TYPES.DATE,
+  KBN_FIELD_TYPES.GEO_POINT,
+];
+
+export const SORT_DIRECTION = {
+  ASC: 'asc',
+  DESC: 'desc',
+} as const;
+
+export type SortDirection = typeof SORT_DIRECTION[keyof typeof SORT_DIRECTION];
+
+export const SORT_MODE = {
+  MIN: 'min',
+  MAX: 'max',
+  AVG: 'avg',
+  SUM: 'sum',
+  MEDIAN: 'median',
+} as const;
+
+export const NUMERIC_TYPES_OPTIONS = {
+  [KBN_FIELD_TYPES.NUMBER]: [ES_FIELD_TYPES.DOUBLE, ES_FIELD_TYPES.LONG],
+  [KBN_FIELD_TYPES.DATE]: [ES_FIELD_TYPES.DATE, ES_FIELD_TYPES.DATE_NANOS],
+};
+
+export type KbnNumericType = typeof KBN_FIELD_TYPES.NUMBER | typeof KBN_FIELD_TYPES.DATE;
+
+const SORT_NUMERIC_FIELD_TYPES = [
+  ES_FIELD_TYPES.DOUBLE,
+  ES_FIELD_TYPES.LONG,
+  ES_FIELD_TYPES.DATE,
+  ES_FIELD_TYPES.DATE_NANOS,
+] as const;
+
+export type SortNumericFieldType = typeof SORT_NUMERIC_FIELD_TYPES[number];
+
+export type SortMode = typeof SORT_MODE[keyof typeof SORT_MODE];
+
+export const TOP_METRICS_SPECIAL_SORT_FIELDS = {
+  _SCORE: '_score',
+} as const;
+
+export const isSpecialSortField = (sortField: unknown) => {
+  return Object.values(TOP_METRICS_SPECIAL_SORT_FIELDS).some((v) => v === sortField);
+};
+
+export const isValidSortDirection = (arg: unknown): arg is SortDirection => {
+  return Object.values(SORT_DIRECTION).some((v) => v === arg);
+};
+
+export const isValidSortMode = (arg: unknown): arg is SortMode => {
+  return Object.values(SORT_MODE).some((v) => v === arg);
+};
+
+export const isValidSortNumericType = (arg: unknown): arg is SortNumericFieldType => {
+  return SORT_NUMERIC_FIELD_TYPES.some((v) => v === arg);
+};
+
 /**
  * The maximum level of sub-aggregations
  */
@@ -75,6 +137,10 @@ export interface PivotAggsConfigBase {
   agg: PivotSupportedAggs;
   aggName: AggName;
   dropDownName: string;
+  /**
+   * Indicates if aggregation supports multiple fields
+   */
+  isMultiField?: boolean;
   /** Indicates if aggregation supports sub-aggregations */
   isSubAggsSupported?: boolean;
   /** Dictionary of the sub-aggregations */
@@ -130,7 +196,7 @@ export function getAggConfigFromEsAgg(
 }
 
 export interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
-  field: EsFieldName;
+  field: EsFieldName | EsFieldName[];
 }
 
 export interface PivotAggsConfigWithExtra<T> extends PivotAggsConfigWithUiBase {

x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/advanced_runtime_mappings_settings.tsx

@@ -46,7 +46,7 @@ export const AdvancedRuntimeMappingsSettings: FC<StepDefineFormHook> = (props) =
     },
   } = props.runtimeMappingsEditor;
   const {
-    actions: { deleteAggregation, deleteGroupBy },
+    actions: { deleteAggregation, deleteGroupBy, updateAggregation },
     state: { groupByList, aggList },
   } = props.pivotConfig;
 
@@ -55,6 +55,9 @@ export const AdvancedRuntimeMappingsSettings: FC<StepDefineFormHook> = (props) =
       advancedRuntimeMappingsConfig === '' ? {} : JSON.parse(advancedRuntimeMappingsConfig);
     const previousConfig = runtimeMappings;
 
+    const isFieldDeleted = (field: string) =>
+      previousConfig?.hasOwnProperty(field) && !nextConfig.hasOwnProperty(field);
+
     applyRuntimeMappingsEditorChanges();
 
     // If the user updates the name of the runtime mapping fields
@@ -71,13 +74,16 @@ export const AdvancedRuntimeMappingsSettings: FC<StepDefineFormHook> = (props) =
     });
     Object.keys(aggList).forEach((aggName) => {
       const agg = aggList[aggName] as PivotAggsConfigWithUiSupport;
-      if (
-        isPivotAggConfigWithUiSupport(agg) &&
-        agg.field !== undefined &&
-        previousConfig?.hasOwnProperty(agg.field) &&
-        !nextConfig.hasOwnProperty(agg.field)
-      ) {
-        deleteAggregation(aggName);
+
+      if (isPivotAggConfigWithUiSupport(agg)) {
+        if (Array.isArray(agg.field)) {
+          const newFields = agg.field.filter((f) => !isFieldDeleted(f));
+          updateAggregation(aggName, { ...agg, field: newFields });
+        } else {
+          if (agg.field !== undefined && isFieldDeleted(agg.field)) {
+            deleteAggregation(aggName);
+          }
+        }
       }
     });
   };

x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx

@@ -12,6 +12,7 @@ import { i18n } from '@kbn/i18n';
 import {
   EuiButton,
   EuiCodeEditor,
+  EuiComboBox,
   EuiFieldText,
   EuiForm,
   EuiFormRow,
@@ -79,7 +80,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
 
   const [aggName, setAggName] = useState(defaultData.aggName);
   const [agg, setAgg] = useState(defaultData.agg);
-  const [field, setField] = useState(
+  const [field, setField] = useState<string | string[]>(
     isPivotAggsConfigWithUiSupport(defaultData) ? defaultData.field : ''
   );
 
@@ -148,13 +149,21 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
 
   if (!isUnsupportedAgg) {
     const optionsArr = dictionaryToArray(options);
+
     optionsArr
       .filter((o) => o.agg === defaultData.agg)
       .forEach((o) => {
         availableFields.push({ text: o.field });
       });
+
     optionsArr
-      .filter((o) => isPivotAggsConfigWithUiSupport(defaultData) && o.field === defaultData.field)
+      .filter(
+        (o) =>
+          isPivotAggsConfigWithUiSupport(defaultData) &&
+          (Array.isArray(defaultData.field)
+            ? defaultData.field.includes(o.field as string)
+            : o.field === defaultData.field)
+      )
       .forEach((o) => {
         availableAggs.push({ text: o.agg });
       });
@@ -217,20 +226,48 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
           data-test-subj="transformAggName"
         />
       </EuiFormRow>
-      {availableFields.length > 0 && (
-        <EuiFormRow
-          label={i18n.translate('xpack.transform.agg.popoverForm.fieldLabel', {
-            defaultMessage: 'Field',
-          })}
-        >
-          <EuiSelect
-            options={availableFields}
-            value={field}
-            onChange={(e) => setField(e.target.value)}
-            data-test-subj="transformAggField"
-          />
-        </EuiFormRow>
-      )}
+      {availableFields.length > 0 ? (
+        aggConfigDef.isMultiField ? (
+          <EuiFormRow
+            label={i18n.translate('xpack.transform.agg.popoverForm.fieldsLabel', {
+              defaultMessage: 'Fields',
+            })}
+          >
+            <EuiComboBox
+              fullWidth
+              options={availableFields.map((v) => {
+                return {
+                  value: v.text,
+                  label: v.text as string,
+                };
+              })}
+              selectedOptions={(typeof field === 'string' ? [field] : field).map((v) => ({
+                value: v,
+                label: v,
+              }))}
+              onChange={(e) => {
+                const res = e.map((v) => v.value as string);
+                setField(res);
+              }}
+              isClearable={false}
+              data-test-subj="transformAggFields"
+            />
+          </EuiFormRow>
+        ) : (
+          <EuiFormRow
+            label={i18n.translate('xpack.transform.agg.popoverForm.fieldLabel', {
+              defaultMessage: 'Field',
+            })}
+          >
+            <EuiSelect
+              options={availableFields}
+              value={field as string}
+              onChange={(e) => setField(e.target.value)}
+              data-test-subj="transformAggField"
+            />
+          </EuiFormRow>
+        )
+      ) : null}
       {availableAggs.length > 0 && (
         <EuiFormRow
           label={i18n.translate('xpack.transform.agg.popoverForm.aggLabel', {
@@ -248,7 +285,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
       {isPivotAggsWithExtendedForm(aggConfigDef) && (
         <aggConfigDef.AggFormComponent
           aggConfig={aggConfigDef.aggConfig}
-          selectedField={field}
+          selectedField={field as string}
           onChange={(update) => {
             setAggConfigDef({
               ...aggConfigDef,

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/common.test.ts

@@ -44,6 +44,7 @@ describe('Transform: Define Pivot Common', () => {
             { label: 'sum( the-f[i]e>ld )' },
             { label: 'value_count( the-f[i]e>ld )' },
             { label: 'filter( the-f[i]e>ld )' },
+            { label: 'top_metrics( the-f[i]e>ld )' },
           ],
         },
       ],
@@ -133,6 +134,7 @@ describe('Transform: Define Pivot Common', () => {
             { label: 'sum( the-f[i]e>ld )' },
             { label: 'value_count( the-f[i]e>ld )' },
             { label: 'filter( the-f[i]e>ld )' },
+            { label: 'top_metrics( the-f[i]e>ld )' },
           ],
         },
         {
@@ -146,6 +148,7 @@ describe('Transform: Define Pivot Common', () => {
             { label: 'sum(rt_bytes_bigger)' },
             { label: 'value_count(rt_bytes_bigger)' },
             { label: 'filter(rt_bytes_bigger)' },
+            { label: 'top_metrics(rt_bytes_bigger)' },
           ],
         },
       ],

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_form_config.ts

@@ -12,6 +12,7 @@ import {
 
 import { PivotAggsConfigBase, PivotAggsConfigWithUiBase } from '../../../../../common/pivot_aggs';
 import { getFilterAggConfig } from './filter_agg/config';
+import { getTopMetricsAggConfig } from './top_metrics_agg/config';
 
 /**
  * Gets form configuration for provided aggregation type.
@@ -23,6 +24,8 @@ export function getAggFormConfig(
   switch (agg) {
     case PIVOT_SUPPORTED_AGGS.FILTER:
       return getFilterAggConfig(commonConfig);
+    case PIVOT_SUPPORTED_AGGS.TOP_METRICS:
+      return getTopMetricsAggConfig(commonConfig);
     default:
       return commonConfig;
   }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_aggregation_config.ts

@@ -15,6 +15,7 @@ import {
   PivotAggsConfigWithUiSupport,
 } from '../../../../../common';
 import { getFilterAggConfig } from './filter_agg/config';
+import { getTopMetricsAggConfig } from './top_metrics_agg/config';
 
 /**
  * Provides a configuration based on the aggregation type.
@@ -41,6 +42,8 @@ export function getDefaultAggregationConfig(
       };
     case PIVOT_SUPPORTED_AGGS.FILTER:
       return getFilterAggConfig(commonConfig);
+    case PIVOT_SUPPORTED_AGGS.TOP_METRICS:
+      return getTopMetricsAggConfig(commonConfig);
     default:
       return commonConfig;
   }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts

@@ -141,6 +141,7 @@ export function getPivotDropdownOptions(
   });
 
   return {
+    fields: combinedFields,
     groupByOptions,
     groupByOptionsData,
     aggOptions,