From 18780bceceb65bb872345db0f65ba993420afdc1 Mon Sep 17 00:00:00 2001
From: Pablo Machado <pablo.nevesmachado@elastic.co>
Date: Tue, 13 Jul 2021 19:27:41 +0200
Subject: [PATCH 01/27] Add empty string validation for Tags and Authors
 (#101756)

---
 .../rules/step_about_rule/schema.tsx          | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
index 3467b34d47135c..1dd59d49e4ff53 100644
--- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
+++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx
@@ -13,6 +13,7 @@ import {
   FormSchema,
   ValidationFunc,
   ERROR_CODE,
+  VALIDATION_TYPES,
 } from '../../../../shared_imports';
 import { AboutStepRule } from '../../../pages/detection_engine/rules/types';
 import { OptionalFieldLabel } from '../optional_field_label';
@@ -38,6 +39,20 @@ export const schema: FormSchema<AboutStepRule> = {
       }
     ),
     labelAppend: OptionalFieldLabel,
+    validations: [
+      {
+        validator: emptyField(
+          i18n.translate(
+            'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.authorFieldEmptyError',
+            {
+              defaultMessage: 'An author must not be empty',
+            }
+          )
+        ),
+        type: VALIDATION_TYPES.ARRAY_ITEM,
+        isBlocking: false,
+      },
+    ],
   },
   name: {
     type: FIELD_TYPES.TEXT,
@@ -243,6 +258,20 @@ export const schema: FormSchema<AboutStepRule> = {
       }
     ),
     labelAppend: OptionalFieldLabel,
+    validations: [
+      {
+        validator: emptyField(
+          i18n.translate(
+            'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.tagFieldEmptyError',
+            {
+              defaultMessage: 'A tag must not be empty',
+            }
+          )
+        ),
+        type: VALIDATION_TYPES.ARRAY_ITEM,
+        isBlocking: false,
+      },
+    ],
   },
   note: {
     type: FIELD_TYPES.TEXTAREA,

From d65743cc3c1b1ba1b08120ecc9310deb64878509 Mon Sep 17 00:00:00 2001
From: Matthew Kime <matt@mattki.me>
Date: Tue, 13 Jul 2021 12:57:14 -0500
Subject: [PATCH 02/27] Revert "Rollup index pattern list fixes (#105328)"
 (#105456)

This reverts commit 6933d630078b2310ebfa73365543f6fb7c347229.
---
 ...in-plugins-data-public.indexpatterntype.md |  19 ---
 ...lugins-data-public.indexpatterntypemeta.md |   1 -
 ...data-public.indexpatterntypemeta.params.md |  13 --
 .../kibana-plugin-plugins-data-public.md      |   1 -
 .../data/common/index_patterns/types.ts       |  10 +-
 src/plugins/data/public/index.ts              |   1 -
 src/plugins/data/public/public.api.md         |  44 ++---
 .../indexed_fields_table.test.tsx.snap        | 155 ------------------
 .../components/table/table.test.tsx           |   4 +-
 .../indexed_fields_table.test.tsx             |  55 +------
 .../indexed_fields_table.tsx                  |   4 +-
 .../public/service/list/config.ts             |  14 +-
 .../public/service/list/manager.ts            |  13 +-
 ...p_list_config.ts => rollup_list_config.js} |  22 +--
 14 files changed, 53 insertions(+), 303 deletions(-)
 delete mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
 delete mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
 rename src/plugins/index_pattern_management/public/service/list/{rollup_list_config.ts => rollup_list_config.js} (69%)

diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
deleted file mode 100644
index 46fd3a0725e403..00000000000000
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
+++ /dev/null
@@ -1,19 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternType](./kibana-plugin-plugins-data-public.indexpatterntype.md)
-
-## IndexPatternType enum
-
-<b>Signature:</b>
-
-```typescript
-export declare enum IndexPatternType 
-```
-
-## Enumeration Members
-
-|  Member | Value | Description |
-|  --- | --- | --- |
-|  DEFAULT | <code>&quot;default&quot;</code> |  |
-|  ROLLUP | <code>&quot;rollup&quot;</code> |  |
-
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
index 19a884862d4609..e6690b244c9ea5 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
@@ -15,5 +15,4 @@ export interface TypeMeta
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [aggs](./kibana-plugin-plugins-data-public.indexpatterntypemeta.aggs.md) | <code>Record&lt;string, AggregationRestrictions&gt;</code> |  |
-|  [params](./kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md) | <code>{</code><br/><code>        rollup_index: string;</code><br/><code>    }</code> |  |
 
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
deleted file mode 100644
index 12646a39188a07..00000000000000
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
+++ /dev/null
@@ -1,13 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternTypeMeta](./kibana-plugin-plugins-data-public.indexpatterntypemeta.md) &gt; [params](./kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md)
-
-## IndexPatternTypeMeta.params property
-
-<b>Signature:</b>
-
-```typescript
-params?: {
-        rollup_index: string;
-    };
-```
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
index 7c2911875ee054..65c4601d5faec9 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
@@ -31,7 +31,6 @@
 |  --- | --- |
 |  [BUCKET\_TYPES](./kibana-plugin-plugins-data-public.bucket_types.md) |  |
 |  [ES\_FIELD\_TYPES](./kibana-plugin-plugins-data-public.es_field_types.md) | \* |
-|  [IndexPatternType](./kibana-plugin-plugins-data-public.indexpatterntype.md) |  |
 |  [KBN\_FIELD\_TYPES](./kibana-plugin-plugins-data-public.kbn_field_types.md) | \* |
 |  [METRIC\_TYPES](./kibana-plugin-plugins-data-public.metric_types.md) |  |
 |  [QuerySuggestionTypes](./kibana-plugin-plugins-data-public.querysuggestiontypes.md) |  |
diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts
index 58cc6d0478d5e8..b03e745df74a6a 100644
--- a/src/plugins/data/common/index_patterns/types.ts
+++ b/src/plugins/data/common/index_patterns/types.ts
@@ -150,17 +150,9 @@ export type AggregationRestrictions = Record<
     time_zone?: string;
   }
 >;
-
 export interface TypeMeta {
   aggs?: Record<string, AggregationRestrictions>;
-  params?: {
-    rollup_index: string;
-  };
-}
-
-export enum IndexPatternType {
-  DEFAULT = 'default',
-  ROLLUP = 'rollup',
+  [key: string]: any;
 }
 
 export type FieldSpecConflictDescriptions = Record<string, string[]>;
diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts
index 9af1cbf95d94da..e9e50ebfaf138c 100644
--- a/src/plugins/data/public/index.ts
+++ b/src/plugins/data/public/index.ts
@@ -269,7 +269,6 @@ export {
   IndexPatternLoadExpressionFunctionDefinition,
   fieldList,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
-  IndexPatternType,
 } from '../common';
 
 export { DuplicateIndexPatternError } from '../common/index_patterns/errors';
diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
index 6c0ddd000f30aa..b8af7c12d57fcc 100644
--- a/src/plugins/data/public/public.api.md
+++ b/src/plugins/data/public/public.api.md
@@ -1688,26 +1688,14 @@ export class IndexPatternsService {
     updateSavedObject(indexPattern: IndexPattern, saveAttempts?: number, ignoreErrors?: boolean): Promise<void | Error>;
 }
 
-// Warning: (ae-missing-release-tag) "IndexPatternType" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
-export enum IndexPatternType {
-    // (undocumented)
-    DEFAULT = "default",
-    // (undocumented)
-    ROLLUP = "rollup"
-}
-
 // Warning: (ae-missing-release-tag) "TypeMeta" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
 export interface IndexPatternTypeMeta {
     // (undocumented)
-    aggs?: Record<string, IndexPatternAggRestrictions>;
+    [key: string]: any;
     // (undocumented)
-    params?: {
-        rollup_index: string;
-    };
+    aggs?: Record<string, IndexPatternAggRestrictions>;
 }
 
 // Warning: (ae-missing-release-tag) "injectReferences" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -2778,20 +2766,20 @@ export interface WaitUntilNextSessionCompletesOptions {
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:412:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:413:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:425:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:430:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:434:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:437:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:411:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:412:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:432:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:436:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:5 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/search/session/session_service.ts:62:5 - (ae-forgotten-export) The symbol "UrlGeneratorStateMapping" needs to be exported by the entry point index.d.ts
 
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
index 776294a93df185..32a2b936edd004 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
@@ -1,146 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`IndexedFieldsTable IndexedFieldsTable with rollup index pattern should render normally 1`] = `
-<div>
-  <Table
-    deleteField={[Function]}
-    editField={[Function]}
-    indexPattern={
-      Object {
-        "getFormatterForFieldNoDefault": [Function],
-        "getNonScriptedFields": [Function],
-        "type": "rollup",
-        "typeMeta": Object {
-          "aggs": Object {
-            "avg": Object {
-              "amount": Object {
-                "agg": "avg",
-              },
-            },
-            "date_histogram": Object {
-              "timestamp": Object {
-                "agg": "date_histogram",
-                "delay": "30s",
-                "fixed_interval": "30s",
-                "time_zone": "UTC",
-              },
-            },
-            "histogram": Object {
-              "amount": Object {
-                "agg": "histogram",
-                "interval": 5,
-              },
-            },
-            "max": Object {
-              "amount": Object {
-                "agg": "max",
-              },
-            },
-            "min": Object {
-              "amount": Object {
-                "agg": "min",
-              },
-            },
-            "sum": Object {
-              "amount": Object {
-                "agg": "sum",
-              },
-            },
-            "terms": Object {
-              "Elastic": Object {
-                "agg": "terms",
-              },
-            },
-            "value_count": Object {
-              "amount": Object {
-                "agg": "value_count",
-              },
-            },
-          },
-          "rollup-index": "rollup",
-        },
-      }
-    }
-    items={
-      Array [
-        Object {
-          "displayName": "Elastic",
-          "esTypes": Array [
-            "keyword",
-          ],
-          "excluded": false,
-          "format": "",
-          "hasRuntime": false,
-          "info": Array [
-            "Rollup aggregations:",
-            "terms",
-          ],
-          "isMapped": false,
-          "kbnType": undefined,
-          "name": "Elastic",
-          "searchable": true,
-          "type": "keyword",
-        },
-        Object {
-          "displayName": "timestamp",
-          "esTypes": Array [
-            "date",
-          ],
-          "excluded": false,
-          "format": "",
-          "hasRuntime": false,
-          "info": Array [
-            "Rollup aggregations:",
-            "date_histogram (interval: 30s, delay: 30s, UTC)",
-          ],
-          "isMapped": false,
-          "kbnType": undefined,
-          "name": "timestamp",
-          "type": "date",
-        },
-        Object {
-          "displayName": "conflictingField",
-          "esTypes": Array [
-            "keyword",
-            "long",
-          ],
-          "excluded": false,
-          "format": "",
-          "hasRuntime": false,
-          "info": Array [],
-          "isMapped": false,
-          "kbnType": undefined,
-          "name": "conflictingField",
-          "type": "keyword, long",
-        },
-        Object {
-          "displayName": "amount",
-          "esTypes": Array [
-            "long",
-          ],
-          "excluded": false,
-          "format": "",
-          "hasRuntime": false,
-          "info": Array [
-            "Rollup aggregations:",
-            "histogram (interval: 5)",
-            "avg",
-            "max",
-            "min",
-            "sum",
-            "value_count",
-          ],
-          "isMapped": false,
-          "kbnType": undefined,
-          "name": "amount",
-          "type": "long",
-        },
-      ]
-    }
-  />
-</div>
-`;
-
 exports[`IndexedFieldsTable should filter based on the query bar 1`] = `
 <div>
   <Table
@@ -265,20 +124,6 @@ exports[`IndexedFieldsTable should render normally 1`] = `
           "name": "conflictingField",
           "type": "keyword, long",
         },
-        Object {
-          "displayName": "amount",
-          "esTypes": Array [
-            "long",
-          ],
-          "excluded": false,
-          "format": "",
-          "hasRuntime": false,
-          "info": Array [],
-          "isMapped": false,
-          "kbnType": undefined,
-          "name": "amount",
-          "type": "long",
-        },
       ]
     }
   />
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
index 163152b52e80b9..b9957534d8c694 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
@@ -8,13 +8,13 @@
 
 import React from 'react';
 import { shallow } from 'enzyme';
-import { IndexPattern } from 'src/plugins/data/public';
+import { IIndexPattern } from 'src/plugins/data/public';
 import { IndexedFieldItem } from '../../types';
 import { Table, renderFieldName } from './table';
 
 const indexPattern = {
   timeFieldName: 'timestamp',
-} as IndexPattern;
+} as IIndexPattern;
 
 const items: IndexedFieldItem[] = [
   {
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
index c819b1d3a33fd9..e587ada6695cb4 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
@@ -8,9 +8,8 @@
 
 import React from 'react';
 import { shallow } from 'enzyme';
-import { IndexPatternField, IndexPattern, IndexPatternType } from 'src/plugins/data/public';
+import { IndexPatternField, IndexPattern } from 'src/plugins/data/public';
 import { IndexedFieldsTable } from './indexed_fields_table';
-import { RollupIndexPatternListConfig } from '../../../service/list';
 
 jest.mock('@elastic/eui', () => ({
   EuiFlexGroup: 'eui-flex-group',
@@ -29,8 +28,7 @@ jest.mock('./components/table', () => ({
 const helpers = {
   editField: (fieldName: string) => {},
   deleteField: (fieldName: string) => {},
-  // getFieldInfo handles non rollups as well
-  getFieldInfo: new RollupIndexPatternListConfig().getFieldInfo,
+  getFieldInfo: () => [],
 };
 
 const indexPattern = ({
@@ -38,32 +36,6 @@ const indexPattern = ({
   getFormatterForFieldNoDefault: () => ({ params: () => ({}) }),
 } as unknown) as IndexPattern;
 
-const rollupIndexPattern = ({
-  type: IndexPatternType.ROLLUP,
-  typeMeta: {
-    'rollup-index': 'rollup',
-    aggs: {
-      date_histogram: {
-        timestamp: {
-          agg: 'date_histogram',
-          fixed_interval: '30s',
-          delay: '30s',
-          time_zone: 'UTC',
-        },
-      },
-      terms: { Elastic: { agg: 'terms' } },
-      histogram: { amount: { agg: 'histogram', interval: 5 } },
-      avg: { amount: { agg: 'avg' } },
-      max: { amount: { agg: 'max' } },
-      min: { amount: { agg: 'min' } },
-      sum: { amount: { agg: 'sum' } },
-      value_count: { amount: { agg: 'value_count' } },
-    },
-  },
-  getNonScriptedFields: () => fields,
-  getFormatterForFieldNoDefault: () => ({ params: () => ({}) }),
-} as unknown) as IndexPattern;
-
 const mockFieldToIndexPatternField = (
   spec: Record<string, string | string[] | boolean | undefined>
 ) => {
@@ -79,7 +51,6 @@ const fields = [
   },
   { name: 'timestamp', displayName: 'timestamp', esTypes: ['date'] },
   { name: 'conflictingField', displayName: 'conflictingField', esTypes: ['keyword', 'long'] },
-  { name: 'amount', displayName: 'amount', esTypes: ['long'] },
 ].map(mockFieldToIndexPatternField);
 
 describe('IndexedFieldsTable', () => {
@@ -144,26 +115,4 @@ describe('IndexedFieldsTable', () => {
 
     expect(component).toMatchSnapshot();
   });
-
-  describe('IndexedFieldsTable with rollup index pattern', () => {
-    test('should render normally', async () => {
-      const component = shallow(
-        <IndexedFieldsTable
-          fields={fields}
-          indexPattern={rollupIndexPattern}
-          helpers={helpers}
-          fieldWildcardMatcher={() => {
-            return () => false;
-          }}
-          indexedFieldTypeFilter=""
-          fieldFilter=""
-        />
-      );
-
-      await new Promise((resolve) => process.nextTick(resolve));
-      component.update();
-
-      expect(component).toMatchSnapshot();
-    });
-  });
 });
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
index 4e9aeb1874c897..ee1147997079f4 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
@@ -8,7 +8,7 @@
 
 import React, { Component } from 'react';
 import { createSelector } from 'reselect';
-import { IndexPatternField, IndexPattern } from '../../../../../../plugins/data/public';
+import { IndexPatternField, IndexPattern, IFieldType } from '../../../../../../plugins/data/public';
 import { Table } from './components/table';
 import { IndexedFieldItem } from './types';
 
@@ -20,7 +20,7 @@ interface IndexedFieldsTableProps {
   helpers: {
     editField: (fieldName: string) => void;
     deleteField: (fieldName: string) => void;
-    getFieldInfo: (indexPattern: IndexPattern, field: IndexPatternField) => string[];
+    getFieldInfo: (indexPattern: IndexPattern, field: IFieldType) => string[];
   };
   fieldWildcardMatcher: (filters: any[]) => (val: any) => boolean;
 }
diff --git a/src/plugins/index_pattern_management/public/service/list/config.ts b/src/plugins/index_pattern_management/public/service/list/config.ts
index 4be27fc47d0db8..e13f8c1c06241e 100644
--- a/src/plugins/index_pattern_management/public/service/list/config.ts
+++ b/src/plugins/index_pattern_management/public/service/list/config.ts
@@ -7,7 +7,8 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { IndexPattern, IndexPatternField, IndexPatternType } from '../../../../data/public';
+import { IIndexPattern, IFieldType } from 'src/plugins/data/public';
+import { SimpleSavedObject } from 'src/core/public';
 
 export interface IndexPatternTag {
   key: string;
@@ -22,9 +23,12 @@ const defaultIndexPatternListName = i18n.translate(
 );
 
 export class IndexPatternListConfig {
-  public readonly key: IndexPatternType = IndexPatternType.DEFAULT;
+  public readonly key = 'default';
 
-  public getIndexPatternTags(indexPattern: IndexPattern, isDefault: boolean): IndexPatternTag[] {
+  public getIndexPatternTags(
+    indexPattern: IIndexPattern | SimpleSavedObject<IIndexPattern>,
+    isDefault: boolean
+  ): IndexPatternTag[] {
     return isDefault
       ? [
           {
@@ -35,11 +39,11 @@ export class IndexPatternListConfig {
       : [];
   }
 
-  public getFieldInfo(indexPattern: IndexPattern, field: IndexPatternField): string[] {
+  public getFieldInfo(indexPattern: IIndexPattern, field: IFieldType): string[] {
     return [];
   }
 
-  public areScriptedFieldsEnabled(indexPattern: IndexPattern): boolean {
+  public areScriptedFieldsEnabled(indexPattern: IIndexPattern): boolean {
     return true;
   }
 }
diff --git a/src/plugins/index_pattern_management/public/service/list/manager.ts b/src/plugins/index_pattern_management/public/service/list/manager.ts
index d9cefbd8001a5d..bdb2d47057f1f2 100644
--- a/src/plugins/index_pattern_management/public/service/list/manager.ts
+++ b/src/plugins/index_pattern_management/public/service/list/manager.ts
@@ -6,11 +6,13 @@
  * Side Public License, v 1.
  */
 
-import { IndexPattern, IndexPatternField } from 'src/plugins/data/public';
+import { IIndexPattern, IFieldType } from 'src/plugins/data/public';
+import { SimpleSavedObject } from 'src/core/public';
 import { once } from 'lodash';
 import { CoreStart } from '../../../../../core/public';
 import { IndexPatternListConfig, IndexPatternTag } from './config';
 import { CONFIG_ROLLUPS } from '../../constants';
+// @ts-ignore
 import { RollupIndexPatternListConfig } from './rollup_list_config';
 
 interface IndexPatternListManagerStart {
@@ -30,7 +32,10 @@ export class IndexPatternListManager {
       return configs;
     });
     return {
-      getIndexPatternTags: (indexPattern: IndexPattern, isDefault: boolean) =>
+      getIndexPatternTags: (
+        indexPattern: IIndexPattern | SimpleSavedObject<IIndexPattern>,
+        isDefault: boolean
+      ) =>
         getConfigs().reduce(
           (tags: IndexPatternTag[], config) =>
             config.getIndexPatternTags
@@ -39,14 +44,14 @@ export class IndexPatternListManager {
           []
         ),
 
-      getFieldInfo: (indexPattern: IndexPattern, field: IndexPatternField): string[] =>
+      getFieldInfo: (indexPattern: IIndexPattern, field: IFieldType): string[] =>
         getConfigs().reduce(
           (info: string[], config) =>
             config.getFieldInfo ? info.concat(config.getFieldInfo(indexPattern, field)) : info,
           []
         ),
 
-      areScriptedFieldsEnabled: (indexPattern: IndexPattern): boolean =>
+      areScriptedFieldsEnabled: (indexPattern: IIndexPattern): boolean =>
         getConfigs().every((config) =>
           config.areScriptedFieldsEnabled ? config.areScriptedFieldsEnabled(indexPattern) : true
         ),
diff --git a/src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts b/src/plugins/index_pattern_management/public/service/list/rollup_list_config.js
similarity index 69%
rename from src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts
rename to src/plugins/index_pattern_management/public/service/list/rollup_list_config.js
index bb9da0d461701d..9a80d5fd0d622b 100644
--- a/src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts
+++ b/src/plugins/index_pattern_management/public/service/list/rollup_list_config.js
@@ -6,17 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { IndexPattern, IndexPatternField, IndexPatternType } from '../../../../data/public';
 import { IndexPatternListConfig } from '.';
 
-function isRollup(indexPattern: IndexPattern) {
-  return indexPattern.type === IndexPatternType.ROLLUP;
+function isRollup(indexPattern) {
+  return (
+    indexPattern.type === 'rollup' || (indexPattern.get && indexPattern.get('type') === 'rollup')
+  );
 }
 
 export class RollupIndexPatternListConfig extends IndexPatternListConfig {
-  key = IndexPatternType.ROLLUP;
+  key = 'rollup';
 
-  getIndexPatternTags = (indexPattern: IndexPattern) => {
+  getIndexPatternTags = (indexPattern) => {
     return isRollup(indexPattern)
       ? [
           {
@@ -27,13 +28,13 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
       : [];
   };
 
-  getFieldInfo = (indexPattern: IndexPattern, field: IndexPatternField) => {
+  getFieldInfo = (indexPattern, field) => {
     if (!isRollup(indexPattern)) {
       return [];
     }
 
     const allAggs = indexPattern.typeMeta && indexPattern.typeMeta.aggs;
-    const fieldAggs = allAggs && Object.keys(allAggs).filter((agg) => allAggs[agg][field.name]);
+    const fieldAggs = allAggs && Object.keys(allAggs).filter((agg) => allAggs[agg][field]);
 
     if (!fieldAggs || !fieldAggs.length) {
       return [];
@@ -41,12 +42,13 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
 
     return ['Rollup aggregations:'].concat(
       fieldAggs.map((aggName) => {
-        const agg = allAggs![aggName][field.name];
+        const agg = allAggs[aggName][field];
         switch (aggName) {
           case 'date_histogram':
-            return `${aggName} (interval: ${agg.fixed_interval}, ${
+            return `${aggName} (interval: ${agg.interval}, ${
               agg.delay ? `delay: ${agg.delay},` : ''
             } ${agg.time_zone})`;
+            break;
           case 'histogram':
             return `${aggName} (interval: ${agg.interval})`;
           default:
@@ -56,7 +58,7 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
     );
   };
 
-  areScriptedFieldsEnabled = (indexPattern: IndexPattern) => {
+  areScriptedFieldsEnabled = (indexPattern) => {
     return !isRollup(indexPattern);
   };
 }

From c88213b095e96462b89c264ca8af03a05c317fd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= <patryk.kopycinski@elastic.co>
Date: Tue, 13 Jul 2021 21:07:23 +0300
Subject: [PATCH 03/27] [Osquery] Fix Saved Query mapping (#105398)

---
 .../saved_queries/saved_queries_dropdown.tsx  | 28 +++++++++++++------
 .../saved_queries/use_update_saved_query.ts   |  2 +-
 .../lib/saved_query/saved_object_mappings.ts  | 12 ++++----
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx
index 30df2267fbfa16..fc7cee2fc804cd 100644
--- a/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx
+++ b/x-pack/plugins/osquery/public/saved_queries/saved_queries_dropdown.tsx
@@ -6,7 +6,7 @@
  */
 
 import { find } from 'lodash/fp';
-import { EuiCodeBlock, EuiFormRow, EuiComboBox, EuiText } from '@elastic/eui';
+import { EuiCodeBlock, EuiFormRow, EuiComboBox, EuiTextColor } from '@elastic/eui';
 import React, {
   forwardRef,
   useCallback,
@@ -19,6 +19,7 @@ import { SimpleSavedObject } from 'kibana/public';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { useHistory, useLocation } from 'react-router-dom';
+import styled from 'styled-components';
 
 import { useSavedQueries } from './use_saved_queries';
 
@@ -26,6 +27,17 @@ export interface SavedQueriesDropdownRef {
   clearSelection: () => void;
 }
 
+const TextTruncate = styled.div`
+  overflow: hidden;
+  text-overflow: ellipsis;
+`;
+
+const StyledEuiCodeBlock = styled(EuiCodeBlock)`
+  .euiCodeBlock__line {
+    white-space: nowrap;
+  }
+`;
+
 interface SavedQueriesDropdownProps {
   disabled?: boolean;
   onChange: (
@@ -88,12 +100,12 @@ const SavedQueriesDropdownComponent = forwardRef<
     ({ value }) => (
       <>
         <strong>{value.id}</strong>
-        <EuiText size="s" color="subdued">
-          <p className="euiTextColor--subdued">{value.description}</p>
-        </EuiText>
-        <EuiCodeBlock language="sql" fontSize="m" paddingSize="s">
-          {value.query}
-        </EuiCodeBlock>
+        <TextTruncate>
+          <EuiTextColor color="subdued">{value.description}</EuiTextColor>
+        </TextTruncate>
+        <StyledEuiCodeBlock language="sql" fontSize="m" paddingSize="s">
+          {value.query.split('\n').join(' ')}
+        </StyledEuiCodeBlock>
       </>
     ),
     []
@@ -145,7 +157,7 @@ const SavedQueriesDropdownComponent = forwardRef<
         selectedOptions={selectedOptions}
         onChange={handleSavedQueryChange}
         renderOption={renderOption}
-        rowHeight={90}
+        rowHeight={110}
       />
     </EuiFormRow>
   );
diff --git a/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts
index 1260413676a4e1..6f4aa517108112 100644
--- a/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts
+++ b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts
@@ -56,7 +56,7 @@ export const useUpdateSavedQuery = ({ savedQueryId }: UseUpdateSavedQueryProps)
           i18n.translate('xpack.osquery.editSavedQuery.successToastMessageText', {
             defaultMessage: 'Successfully updated "{savedQueryName}" query',
             values: {
-              savedQueryName: payload.attributes?.name ?? '',
+              savedQueryName: payload.attributes?.id ?? '',
             },
           })
         );
diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
index 537b6d7874ab8a..5535d707cf5c0a 100644
--- a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
+++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
@@ -24,7 +24,7 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'date',
     },
     created_by: {
-      type: 'text',
+      type: 'keyword',
     },
     platform: {
       type: 'keyword',
@@ -36,7 +36,7 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'date',
     },
     updated_by: {
-      type: 'text',
+      type: 'keyword',
     },
     interval: {
       type: 'keyword',
@@ -57,19 +57,19 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'text',
     },
     name: {
-      type: 'text',
+      type: 'keyword',
     },
     created_at: {
       type: 'date',
     },
     created_by: {
-      type: 'text',
+      type: 'keyword',
     },
     updated_at: {
       type: 'date',
     },
     updated_by: {
-      type: 'text',
+      type: 'keyword',
     },
     queries: {
       properties: {
@@ -77,7 +77,7 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = {
           type: 'keyword',
         },
         interval: {
-          type: 'text',
+          type: 'keyword',
         },
       },
     },

From 1fdcf367a90c914830d340395ff6f6f3bcaf07cb Mon Sep 17 00:00:00 2001
From: James Gowdy <jgowdy@elastic.co>
Date: Tue, 13 Jul 2021 19:11:28 +0100
Subject: [PATCH 04/27] [ML] Improve bucket span estimator error toast
 (#105457)

---
 .../bucket_span_estimator/estimate_bucket_span.ts    | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts
index 67673901494c7d..88ed17cba00035 100644
--- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts
+++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts
@@ -6,7 +6,7 @@
  */
 
 import { useContext, useState } from 'react';
-
+import { i18n } from '@kbn/i18n';
 import { JobCreatorContext } from '../../../job_creator_context';
 import { EVENT_RATE_FIELD_ID } from '../../../../../../../../../common/types/fields';
 import { BucketSpanEstimatorData } from '../../../../../../../../../common/types/job_service';
@@ -76,10 +76,16 @@ export function useEstimateBucketSpan() {
 
   async function estimateBucketSpan() {
     setStatus(ESTIMATE_STATUS.RUNNING);
-    const { name, error, message } = await ml.estimateBucketSpan(data);
+    const { name, error, message: text } = await ml.estimateBucketSpan(data);
     setStatus(ESTIMATE_STATUS.NOT_RUNNING);
     if (error === true) {
-      getToastNotificationService().displayErrorToast(message);
+      const title = i18n.translate(
+        'xpack.ml.newJob.wizard.pickFieldsStep.bucketSpanEstimator.errorTitle',
+        {
+          defaultMessage: 'Bucket span could not be estimated',
+        }
+      );
+      getToastNotificationService().displayWarningToast({ title, text });
     } else {
       jobCreator.bucketSpan = name;
       jobCreatorUpdate();

From e300a75e4546779c25ac62462dc51d9cce410b45 Mon Sep 17 00:00:00 2001
From: James Gowdy <jgowdy@elastic.co>
Date: Tue, 13 Jul 2021 19:11:55 +0100
Subject: [PATCH 05/27] [ML] Fix calendar creation during model snapshot
 restore (#105421)

* [ML] Fix calendar creation during model snapshot restore

* adding toast for unexpected errors
---
 .../revert_model_snapshot_flyout.tsx                         | 5 +++++
 .../plugins/ml/server/models/job_service/model_snapshots.ts  | 1 -
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/revert_model_snapshot_flyout.tsx b/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/revert_model_snapshot_flyout.tsx
index 6dd4e6c14589b2..f282b2fde2b3ac 100644
--- a/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/revert_model_snapshot_flyout.tsx
+++ b/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/revert_model_snapshot_flyout.tsx
@@ -42,6 +42,7 @@ import { Anomaly } from '../../../jobs/new_job/common/results_loader/results_loa
 import { parseInterval } from '../../../../../common/util/parse_interval';
 import { CreateCalendar, CalendarEvent } from './create_calendar';
 import { timeFormatter } from '../../../../../common/util/date_utils';
+import { toastNotificationServiceProvider } from '../../../services/toast_notification_service';
 
 interface Props {
   snapshot: ModelSnapshot;
@@ -139,6 +140,10 @@ export const RevertModelSnapshotFlyout: FC<Props> = ({
             })
           );
           refresh();
+        })
+        .catch((error) => {
+          const { displayErrorToast } = toastNotificationServiceProvider(toasts);
+          displayErrorToast(error);
         });
       hideRevertModal();
       closeFlyout();
diff --git a/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts b/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts
index 6cb5f67149fb63..56221f9a72c89a 100644
--- a/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts
+++ b/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts
@@ -85,7 +85,6 @@ export function modelSnapshotProvider(client: IScopedClusterClient, mlClient: Ml
           ),
           events: calendarEvents.map((s) => ({
             calendar_id: calendarId,
-            event_id: '',
             description: s.description,
             start_time: `${s.start}`,
             end_time: `${s.end}`,

From de58b8cc0ce402397a8c93893361e79203665ce8 Mon Sep 17 00:00:00 2001
From: Wylie Conlon <william.conlon@elastic.co>
Date: Tue, 13 Jul 2021 14:19:49 -0400
Subject: [PATCH 06/27] [Lens] Fix field stats when multiple runtime fields are
 used (#105359)

---
 .../plugins/lens/server/routes/field_stats.ts | 20 ++++++++----
 .../api_integration/apis/lens/field_stats.ts  | 32 +++++++++++++++++++
 .../es_archives/visualize/default/data.json   |  2 +-
 3 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts
index 12d3ef3f4a95e7..7103e395eabdc3 100644
--- a/x-pack/plugins/lens/server/routes/field_stats.ts
+++ b/x-pack/plugins/lens/server/routes/field_stats.ts
@@ -8,7 +8,7 @@ import { errors, estypes } from '@elastic/elasticsearch';
 import DateMath from '@elastic/datemath';
 import { schema } from '@kbn/config-schema';
 import { CoreSetup } from 'src/core/server';
-import { IFieldType } from 'src/plugins/data/common';
+import type { IndexPatternField } from 'src/plugins/data/common';
 import { SavedObjectNotFound } from '../../../../../src/plugins/kibana_utils/common';
 import { ESSearchResponse } from '../../../../../src/core/types/elasticsearch';
 import { FieldStatsResponse, BASE_API_URL } from '../../common';
@@ -79,6 +79,14 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
           },
         };
 
+        const runtimeMappings = indexPattern.fields
+          .filter((f) => f.runtimeField)
+          .reduce((acc, f) => {
+            if (!f.runtimeField) return acc;
+            acc[f.name] = f.runtimeField;
+            return acc;
+          }, {} as Record<string, estypes.MappingRuntimeField>);
+
         const search = async (aggs: Record<string, estypes.AggregationsAggregationContainer>) => {
           const { body: result } = await requestClient.search({
             index: indexPattern.title,
@@ -86,7 +94,7 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
             body: {
               query,
               aggs,
-              runtime_mappings: field.runtimeField ? { [fieldName]: field.runtimeField } : {},
+              runtime_mappings: runtimeMappings,
             },
             size: 0,
           });
@@ -138,7 +146,7 @@ export async function getNumberHistogram(
   aggSearchWithBody: (
     aggs: Record<string, estypes.AggregationsAggregationContainer>
   ) => Promise<unknown>,
-  field: IFieldType,
+  field: IndexPatternField,
   useTopHits = true
 ): Promise<FieldStatsResponse> {
   const fieldRef = getFieldRef(field);
@@ -247,7 +255,7 @@ export async function getNumberHistogram(
 
 export async function getStringSamples(
   aggSearchWithBody: (aggs: Record<string, estypes.AggregationsAggregationContainer>) => unknown,
-  field: IFieldType,
+  field: IndexPatternField,
   size = 10
 ): Promise<FieldStatsResponse> {
   const fieldRef = getFieldRef(field);
@@ -287,7 +295,7 @@ export async function getStringSamples(
 // This one is not sampled so that it returns the full date range
 export async function getDateHistogram(
   aggSearchWithBody: (aggs: Record<string, estypes.AggregationsAggregationContainer>) => unknown,
-  field: IFieldType,
+  field: IndexPatternField,
   range: { fromDate: string; toDate: string }
 ): Promise<FieldStatsResponse> {
   const fromDate = DateMath.parse(range.fromDate);
@@ -329,7 +337,7 @@ export async function getDateHistogram(
   };
 }
 
-function getFieldRef(field: IFieldType) {
+function getFieldRef(field: IndexPatternField) {
   return field.scripted
     ? {
         script: {
diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts
index 5dcb749f54b317..5090fe14576d58 100644
--- a/x-pack/test/api_integration/apis/lens/field_stats.ts
+++ b/x-pack/test/api_integration/apis/lens/field_stats.ts
@@ -427,6 +427,38 @@ export default ({ getService }: FtrProviderContext) => {
 
         expect(body.totalDocuments).to.eql(425);
       });
+
+      it('should allow filtering on a runtime field other than the field in use', async () => {
+        const { body } = await supertest
+          .post('/api/lens/index_stats/logstash-2015.09.22/field')
+          .set(COMMON_HEADERS)
+          .send({
+            dslQuery: {
+              bool: {
+                filter: [{ exists: { field: 'runtime_string_field' } }],
+              },
+            },
+            fromDate: TEST_START_TIME,
+            toDate: TEST_END_TIME,
+            fieldName: 'runtime_number_field',
+          })
+          .expect(200);
+
+        expect(body).to.eql({
+          totalDocuments: 4634,
+          sampledDocuments: 4634,
+          sampledValues: 4634,
+          topValues: {
+            buckets: [
+              {
+                count: 4634,
+                key: 5,
+              },
+            ],
+          },
+          histogram: { buckets: [] },
+        });
+      });
     });
 
     describe('histogram', () => {
diff --git a/x-pack/test/functional/es_archives/visualize/default/data.json b/x-pack/test/functional/es_archives/visualize/default/data.json
index 7d0ad0c25f96db..a16e1676611ce5 100644
--- a/x-pack/test/functional/es_archives/visualize/default/data.json
+++ b/x-pack/test/functional/es_archives/visualize/default/data.json
@@ -157,7 +157,7 @@
         "timeFieldName": "@timestamp",
         "title": "logstash-2015.09.22",
         "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]",
-        "runtimeFieldMap":"{\"runtime_string_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('hello world!')\"}}}"
+        "runtimeFieldMap":"{\"runtime_string_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('hello world!')\"}},\"runtime_number_field\":{\"type\":\"double\",\"script\":{\"source\":\"emit(5)\"}}}"
       },
       "migrationVersion": {
         "index-pattern": "7.11.0"

From 9c2effc93edaf45167a6dc6f998b2556a0f4ccfb Mon Sep 17 00:00:00 2001
From: Wylie Conlon <william.conlon@elastic.co>
Date: Tue, 13 Jul 2021 15:10:25 -0400
Subject: [PATCH 07/27] [Lens] Fix flaky formula test (#105295)

* [Lens] Fix flaky formula test

* Only run formula tests

* Remove .only

* Increase debounce time
---
 x-pack/test/functional/apps/lens/formula.ts   | 26 ++++++++-----------
 .../test/functional/page_objects/lens_page.ts | 11 ++++++--
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts
index 6148215d8b6d25..7662b32b8aee62 100644
--- a/x-pack/test/functional/apps/lens/formula.ts
+++ b/x-pack/test/functional/apps/lens/formula.ts
@@ -15,9 +15,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const browser = getService('browser');
   const testSubjects = getService('testSubjects');
   const fieldEditor = getService('fieldEditor');
+  const retry = getService('retry');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/105016
-  describe.skip('lens formula', () => {
+  describe('lens formula', () => {
     it('should transition from count to formula', async () => {
       await PageObjects.visualize.gotoVisualizationLandingPage();
       await listingTable.searchForItemWithName('lnsXYvis');
@@ -55,8 +55,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       const input = await find.activeElement();
       await input.type('*');
 
-      await PageObjects.header.waitUntilLoadingHasFinished();
-      expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14,005');
+      await retry.try(async () => {
+        expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14,005');
+      });
     });
 
     it('should insert single quotes and escape when needed to create valid KQL', async () => {
@@ -79,15 +80,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
       await PageObjects.common.sleep(100);
 
-      let element = await find.byCssSelector('.monaco-editor');
-      expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s Clothing ')`);
+      PageObjects.lens.expectFormulaText(`count(kql='Men\\'s Clothing ')`);
 
       await PageObjects.lens.typeFormula('count(kql=');
+
       input = await find.activeElement();
       await input.type(`Men\'s Clothing`);
 
-      element = await find.byCssSelector('.monaco-editor');
-      expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s Clothing')`);
+      PageObjects.lens.expectFormulaText(`count(kql='Men\\'s Clothing')`);
     });
 
     it('should insert single quotes and escape when needed to create valid field name', async () => {
@@ -109,20 +109,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       });
 
       await PageObjects.lens.switchToFormula();
-      let element = await find.byCssSelector('.monaco-editor');
-      expect(await element.getVisibleText()).to.equal(`unique_count('*\\' "\\'')`);
+      PageObjects.lens.expectFormulaText(`unique_count('*\\' "\\'')`);
 
+      await PageObjects.lens.typeFormula('unique_count(');
       const input = await find.activeElement();
-      await input.clearValueWithKeyboard({ charByChar: true });
-      await input.type('unique_count(');
-      await PageObjects.common.sleep(100);
       await input.type('*');
       await input.pressKeys(browser.keys.ENTER);
 
       await PageObjects.common.sleep(100);
 
-      element = await find.byCssSelector('.monaco-editor');
-      expect(await element.getVisibleText()).to.equal(`unique_count('*\\' "\\'')`);
+      PageObjects.lens.expectFormulaText(`unique_count('*\\' "\\'')`);
     });
 
     it('should persist a broken formula on close', async () => {
diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts
index e256d5cd651cc1..1acddd4641ff41 100644
--- a/x-pack/test/functional/page_objects/lens_page.ts
+++ b/x-pack/test/functional/page_objects/lens_page.ts
@@ -139,6 +139,8 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont
       }
 
       if (opts.formula) {
+        // Formula takes time to open
+        await PageObjects.common.sleep(500);
         await this.typeFormula(opts.formula);
       }
 
@@ -1067,13 +1069,18 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont
     },
 
     async typeFormula(formula: string) {
-      // Formula takes time to open
-      await PageObjects.common.sleep(500);
       await find.byCssSelector('.monaco-editor');
       await find.clickByCssSelectorWhenNotDisabled('.monaco-editor');
       const input = await find.activeElement();
       await input.clearValueWithKeyboard({ charByChar: true });
       await input.type(formula);
+      // Debounce time for formula
+      await PageObjects.common.sleep(300);
+    },
+
+    async expectFormulaText(formula: string) {
+      const element = await find.byCssSelector('.monaco-editor');
+      expect(await element.getVisibleText()).to.equal(formula);
     },
 
     async filterLegend(value: string) {

From e5c3065adf4a15758d10778133a2e459853b8dc2 Mon Sep 17 00:00:00 2001
From: Scotty Bollinger <scotty.bollinger@elastic.co>
Date: Tue, 13 Jul 2021 14:36:57 -0500
Subject: [PATCH 08/27] Handle edge case for GitHub oAuth URL mismatch
 (#105302)

* Hangle edge case for github oauth mismatch

* Fix grammar

* Refactor test
---
 .../add_source/add_source_logic.test.ts       | 37 ++++++++++++++++++-
 .../components/add_source/add_source_logic.ts | 27 +++++++++++++-
 .../views/content_sources/constants.ts        |  7 ++++
 3 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.test.ts
index fcaa847c47f3eb..09ba41f81d76ac 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.test.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.test.ts
@@ -20,8 +20,14 @@ jest.mock('../../../../app_logic', () => ({
 }));
 import { AppLogic } from '../../../../app_logic';
 
-import { ADD_GITHUB_PATH, SOURCES_PATH, getSourcesPath } from '../../../../routes';
+import {
+  ADD_GITHUB_PATH,
+  SOURCES_PATH,
+  PERSONAL_SOURCES_PATH,
+  getSourcesPath,
+} from '../../../../routes';
 import { CustomSource } from '../../../../types';
+import { PERSONAL_DASHBOARD_SOURCE_ERROR } from '../../constants';
 import { SourcesLogic } from '../../sources_logic';
 
 import {
@@ -36,7 +42,7 @@ describe('AddSourceLogic', () => {
   const { mount } = new LogicMounter(AddSourceLogic);
   const { http } = mockHttpValues;
   const { navigateToUrl } = mockKibanaValues;
-  const { clearFlashMessages, flashAPIErrors } = mockFlashMessageHelpers;
+  const { clearFlashMessages, flashAPIErrors, setErrorMessage } = mockFlashMessageHelpers;
 
   const defaultValues = {
     addSourceCurrentStep: AddSourceSteps.ConfigIntroStep,
@@ -353,6 +359,33 @@ describe('AddSourceLogic', () => {
         expect(navigateToUrl).toHaveBeenCalledWith(`${ADD_GITHUB_PATH}/configure${queryString}`);
       });
 
+      describe('Github error edge case', () => {
+        const getGithubQueryString = (context: 'organization' | 'account') =>
+          `?error=redirect_uri_mismatch&error_description=The+redirect_uri+MUST+match+the+registered+callback+URL+for+this+application.&error_uri=https%3A%2F%2Fdocs.github.com%2Fapps%2Fmanaging-oauth-apps%2Ftroubleshooting-authorization-request-errors%2F%23redirect-uri-mismatch&state=%7B%22action%22%3A%22create%22%2C%22context%22%3A%22${context}%22%2C%22service_type%22%3A%22github%22%2C%22csrf_token%22%3A%22TOKEN%3D%3D%22%2C%22index_permissions%22%3Afalse%7D`;
+
+        it('handles "organization" redirect and displays error', () => {
+          const githubQueryString = getGithubQueryString('organization');
+          AddSourceLogic.actions.saveSourceParams(githubQueryString);
+
+          expect(navigateToUrl).toHaveBeenCalledWith('/');
+          expect(setErrorMessage).toHaveBeenCalledWith(
+            'The redirect_uri MUST match the registered callback URL for this application.'
+          );
+        });
+
+        it('handles "account" redirect and displays error', () => {
+          const githubQueryString = getGithubQueryString('account');
+          AddSourceLogic.actions.saveSourceParams(githubQueryString);
+
+          expect(navigateToUrl).toHaveBeenCalledWith(PERSONAL_SOURCES_PATH);
+          expect(setErrorMessage).toHaveBeenCalledWith(
+            PERSONAL_DASHBOARD_SOURCE_ERROR(
+              'The redirect_uri MUST match the registered callback URL for this application.'
+            )
+          );
+        });
+      });
+
       it('handles error', async () => {
         http.get.mockReturnValue(Promise.reject('this is an error'));
 
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.ts
index 0bd37aed81c32a..81e27f07293dc2 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/add_source/add_source_logic.ts
@@ -16,14 +16,21 @@ import {
   flashAPIErrors,
   setSuccessMessage,
   clearFlashMessages,
+  setErrorMessage,
 } from '../../../../../shared/flash_messages';
 import { HttpLogic } from '../../../../../shared/http';
 import { KibanaLogic } from '../../../../../shared/kibana';
 import { parseQueryParams } from '../../../../../shared/query_params';
 import { AppLogic } from '../../../../app_logic';
 import { CUSTOM_SERVICE_TYPE, WORKPLACE_SEARCH_URL_PREFIX } from '../../../../constants';
-import { SOURCES_PATH, ADD_GITHUB_PATH, getSourcesPath } from '../../../../routes';
+import {
+  SOURCES_PATH,
+  ADD_GITHUB_PATH,
+  PERSONAL_SOURCES_PATH,
+  getSourcesPath,
+} from '../../../../routes';
 import { CustomSource } from '../../../../types';
+import { PERSONAL_DASHBOARD_SOURCE_ERROR } from '../../constants';
 import { staticSourceData } from '../../source_data';
 import { SourcesLogic } from '../../sources_logic';
 
@@ -50,6 +57,8 @@ export interface OauthParams {
   state: string;
   session_state: string;
   oauth_verifier?: string;
+  error?: string;
+  error_description?: string;
 }
 
 export interface AddSourceActions {
@@ -501,6 +510,22 @@ export const AddSourceLogic = kea<MakeLogicType<AddSourceValues, AddSourceAction
       const state = JSON.parse(params.state);
       const isOrganization = state.context !== 'account';
 
+      /**
+        There is an extreme edge case where the user is trying to connect Github as source from ent-search,
+        after configuring it in Kibana. When this happens, Github redirects the user from ent-search to Kibana
+        with special error properties in the query params. In this case we need to redirect the user to the
+        app home page and display the error message, and not persist the other query params to the server.
+      */
+      if (params.error_description) {
+        navigateToUrl(isOrganization ? '/' : PERSONAL_SOURCES_PATH);
+        setErrorMessage(
+          isOrganization
+            ? params.error_description
+            : PERSONAL_DASHBOARD_SOURCE_ERROR(params.error_description)
+        );
+        return;
+      }
+
       try {
         const response = await http.get(route, { query });
         const {
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts
index 8e5123dbc7cceb..d57afc6699c1af 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/constants.ts
@@ -470,3 +470,10 @@ export const PRIVATE_DASHBOARD_READ_ONLY_MODE_WARNING = i18n.translate(
       'Workplace Search is currently available for search only, due to regular maintenance. Contact your system administrator for more information.',
   }
 );
+
+export const PERSONAL_DASHBOARD_SOURCE_ERROR = (error: string) =>
+  i18n.translate('xpack.enterpriseSearch.workplaceSearch.personalDashboardSourceError', {
+    defaultMessage:
+      'Could not connect the source, reach out to your admin for help. Error message: {error}',
+    values: { error },
+  });

From 5dc1c8f53adddc84bbd1e5b6c113ab745495e33e Mon Sep 17 00:00:00 2001
From: Walter Rafelsberger <walter@elastic.co>
Date: Tue, 13 Jul 2021 22:46:14 +0200
Subject: [PATCH 09/27] [ML] API integration tests for APM latency correlation.
 (#104644)

Adds API integration tests for APM Latency Correlations code.

Writing the tests surfaced some glitches fixed as part of this PR:
- If the applied filters don't return any docs, we won't throw an error anymore. Instead, the async search service finishes early and just returns no results.
- If for whatever reason the async search service throws an error, it will also set its state now to isRunning = false.
- If the client triggers a request with a service ID we now make sure that async search service still exists. We throw an error if that service no longer exists. This avoids re-instantiating async search services when they've already finished or failed and for whatever reason a client triggers another request with the same ID.
- Refactored requests to reuse APM's own getCorrelationsFilters(). We now require start/end to be set and it will be converted from ISO (client side) to epochmillis (server side) to be more in line with APM's existing code.
- The async search service now creates a simple internal log. This gets exposed via the API and we assert it using the API tests. In the future, we might also expose it in the UI to allow for better problem investigation for users and support.
---
 .../correlations/ml_latency_correlations.tsx  |  34 ++-
 .../app/correlations/use_correlations.ts      |   1 +
 .../correlations/async_search_service.ts      |  66 ++++-
 .../get_query_with_params.test.ts             |  55 +++-
 .../correlations/get_query_with_params.ts     |  65 +++--
 .../correlations/query_correlation.test.ts    |   2 +-
 .../query_field_candidates.test.ts            |  11 +-
 .../query_field_value_pairs.test.ts           |   2 +-
 .../correlations/query_fractions.test.ts      |   2 +-
 .../correlations/query_histogram.test.ts      |  11 +-
 .../query_histogram_interval.test.ts          |  11 +-
 ...ts => query_histogram_range_steps.test.ts} |  22 +-
 ...teps.ts => query_histogram_range_steps.ts} |  25 +-
 .../correlations/query_percentiles.test.ts    |  16 +-
 .../correlations/query_percentiles.ts         |  21 +-
 .../correlations/query_ranges.test.ts         |  11 +-
 .../correlations/search_strategy.test.ts      |  30 +-
 .../correlations/search_strategy.ts           |  35 ++-
 .../utils/aggregation_utils.test.ts           |  94 +++++++
 .../tests/correlations/latency_ml.ts          | 266 ++++++++++++++++++
 .../test/apm_api_integration/tests/index.ts   |   4 +
 21 files changed, 675 insertions(+), 109 deletions(-)
 rename x-pack/plugins/apm/server/lib/search_strategies/correlations/{query_histogram_rangesteps.test.ts => query_histogram_range_steps.test.ts} (77%)
 rename x-pack/plugins/apm/server/lib/search_strategies/correlations/{query_histogram_rangesteps.ts => query_histogram_range_steps.ts} (83%)
 create mode 100644 x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts

diff --git a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
index 4bd20f51977c6c..03fab3e788639a 100644
--- a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
+++ b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
@@ -61,17 +61,16 @@ export function MlLatencyCorrelations({ onClose }: Props) {
   } = useApmPluginContext();
 
   const { serviceName } = useParams<{ serviceName: string }>();
-  const { urlParams } = useUrlParams();
-
-  const fetchOptions = useMemo(
-    () => ({
-      ...{
-        serviceName,
-        ...urlParams,
-      },
-    }),
-    [serviceName, urlParams]
-  );
+  const {
+    urlParams: {
+      environment,
+      kuery,
+      transactionName,
+      transactionType,
+      start,
+      end,
+    },
+  } = useUrlParams();
 
   const {
     error,
@@ -85,7 +84,15 @@ export function MlLatencyCorrelations({ onClose }: Props) {
   } = useCorrelations({
     index: 'apm-*',
     ...{
-      ...fetchOptions,
+      ...{
+        environment,
+        kuery,
+        serviceName,
+        transactionName,
+        transactionType,
+        start,
+        end,
+      },
       percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD,
     },
   });
@@ -322,8 +329,7 @@ export function MlLatencyCorrelations({ onClose }: Props) {
                 {
                   defaultMessage: 'Latency distribution for {name}',
                   values: {
-                    name:
-                      fetchOptions.transactionName ?? fetchOptions.serviceName,
+                    name: transactionName ?? serviceName,
                   },
                 }
               )}
diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
index 8c874571d23dba..2baeb63fa4a239 100644
--- a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
+++ b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
@@ -36,6 +36,7 @@ interface RawResponse {
   took: number;
   values: SearchServiceValue[];
   overallHistogram: HistogramItem[];
+  log: string[];
 }
 
 export const useCorrelations = (params: CorrelationsOptions) => {
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
index 7a511fc60fd064..155cb1f4615bdc 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
@@ -11,7 +11,7 @@ import { fetchTransactionDurationFieldCandidates } from './query_field_candidate
 import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
 import { fetchTransactionDurationPercentiles } from './query_percentiles';
 import { fetchTransactionDurationCorrelation } from './query_correlation';
-import { fetchTransactionDurationHistogramRangesteps } from './query_histogram_rangesteps';
+import { fetchTransactionDurationHistogramRangeSteps } from './query_histogram_range_steps';
 import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
 import type {
   AsyncSearchProviderProgress,
@@ -24,6 +24,8 @@ import { fetchTransactionDurationFractions } from './query_fractions';
 const CORRELATION_THRESHOLD = 0.3;
 const KS_TEST_THRESHOLD = 0.1;
 
+const currentTimeAsString = () => new Date().toISOString();
+
 export const asyncSearchServiceProvider = (
   esClient: ElasticsearchClient,
   params: SearchServiceParams
@@ -31,6 +33,9 @@ export const asyncSearchServiceProvider = (
   let isCancelled = false;
   let isRunning = true;
   let error: Error;
+  const log: string[] = [];
+  const logMessage = (message: string) =>
+    log.push(`${currentTimeAsString()}: ${message}`);
 
   const progress: AsyncSearchProviderProgress = {
     started: Date.now(),
@@ -53,13 +58,17 @@ export const asyncSearchServiceProvider = (
   let percentileThresholdValue: number;
 
   const cancel = () => {
+    logMessage(`Service cancelled.`);
     isCancelled = true;
   };
 
   const fetchCorrelations = async () => {
     try {
       // 95th percentile to be displayed as a marker in the log log chart
-      const percentileThreshold = await fetchTransactionDurationPercentiles(
+      const {
+        totalDocs,
+        percentiles: percentileThreshold,
+      } = await fetchTransactionDurationPercentiles(
         esClient,
         params,
         params.percentileThreshold ? [params.percentileThreshold] : undefined
@@ -67,12 +76,32 @@ export const asyncSearchServiceProvider = (
       percentileThresholdValue =
         percentileThreshold[`${params.percentileThreshold}.0`];
 
-      const histogramRangeSteps = await fetchTransactionDurationHistogramRangesteps(
+      logMessage(
+        `Fetched ${params.percentileThreshold}th percentile value of ${percentileThresholdValue} based on ${totalDocs} documents.`
+      );
+
+      // finish early if we weren't able to identify the percentileThresholdValue.
+      if (percentileThresholdValue === undefined) {
+        logMessage(
+          `Abort service since percentileThresholdValue could not be determined.`
+        );
+        progress.loadedHistogramStepsize = 1;
+        progress.loadedOverallHistogram = 1;
+        progress.loadedFieldCanditates = 1;
+        progress.loadedFieldValuePairs = 1;
+        progress.loadedHistograms = 1;
+        isRunning = false;
+        return;
+      }
+
+      const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps(
         esClient,
         params
       );
       progress.loadedHistogramStepsize = 1;
 
+      logMessage(`Loaded histogram range steps.`);
+
       if (isCancelled) {
         isRunning = false;
         return;
@@ -86,6 +115,8 @@ export const asyncSearchServiceProvider = (
       progress.loadedOverallHistogram = 1;
       overallHistogram = overallLogHistogramChartData;
 
+      logMessage(`Loaded overall histogram chart data.`);
+
       if (isCancelled) {
         isRunning = false;
         return;
@@ -93,13 +124,13 @@ export const asyncSearchServiceProvider = (
 
       // Create an array of ranges [2, 4, 6, ..., 98]
       const percents = Array.from(range(2, 100, 2));
-      const percentilesRecords = await fetchTransactionDurationPercentiles(
-        esClient,
-        params,
-        percents
-      );
+      const {
+        percentiles: percentilesRecords,
+      } = await fetchTransactionDurationPercentiles(esClient, params, percents);
       const percentiles = Object.values(percentilesRecords);
 
+      logMessage(`Loaded percentiles.`);
+
       if (isCancelled) {
         isRunning = false;
         return;
@@ -110,6 +141,8 @@ export const asyncSearchServiceProvider = (
         params
       );
 
+      logMessage(`Identified ${fieldCandidates.length} fieldCandidates.`);
+
       progress.loadedFieldCanditates = 1;
 
       const fieldValuePairs = await fetchTransactionDurationFieldValuePairs(
@@ -119,6 +152,8 @@ export const asyncSearchServiceProvider = (
         progress
       );
 
+      logMessage(`Identified ${fieldValuePairs.length} fieldValuePairs.`);
+
       if (isCancelled) {
         isRunning = false;
         return;
@@ -133,6 +168,8 @@ export const asyncSearchServiceProvider = (
         totalDocCount,
       } = await fetchTransactionDurationFractions(esClient, params, ranges);
 
+      logMessage(`Loaded fractions and totalDocCount of ${totalDocCount}.`);
+
       async function* fetchTransactionDurationHistograms() {
         for (const item of shuffle(fieldValuePairs)) {
           if (item === undefined || isCancelled) {
@@ -185,7 +222,11 @@ export const asyncSearchServiceProvider = (
               yield undefined;
             }
           } catch (e) {
-            error = e;
+            // don't fail the whole process for individual correlation queries, just add the error to the internal log.
+            logMessage(
+              `Failed to fetch correlation/kstest for '${item.field}/${item.value}'`
+            );
+            yield undefined;
           }
         }
       }
@@ -199,10 +240,14 @@ export const asyncSearchServiceProvider = (
         progress.loadedHistograms = loadedHistograms / fieldValuePairs.length;
       }
 
-      isRunning = false;
+      logMessage(
+        `Identified ${values.length} significant correlations out of ${fieldValuePairs.length} field/value pairs.`
+      );
     } catch (e) {
       error = e;
     }
+
+    isRunning = false;
   };
 
   fetchCorrelations();
@@ -212,6 +257,7 @@ export const asyncSearchServiceProvider = (
 
     return {
       error,
+      log,
       isRunning,
       loaded: Math.round(progress.getOverallProgress() * 100),
       overallHistogram,
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts
index 12e897ab3eec92..016355b3a64159 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts
@@ -10,10 +10,23 @@ import { getQueryWithParams } from './get_query_with_params';
 describe('correlations', () => {
   describe('getQueryWithParams', () => {
     it('returns the most basic query filtering on processor.event=transaction', () => {
-      const query = getQueryWithParams({ params: { index: 'apm-*' } });
+      const query = getQueryWithParams({
+        params: { index: 'apm-*', start: '2020', end: '2021' },
+      });
       expect(query).toEqual({
         bool: {
-          filter: [{ term: { 'processor.event': 'transaction' } }],
+          filter: [
+            { term: { 'processor.event': 'transaction' } },
+            {
+              range: {
+                '@timestamp': {
+                  format: 'epoch_millis',
+                  gte: 1577836800000,
+                  lte: 1609459200000,
+                },
+              },
+            },
+          ],
         },
       });
     });
@@ -24,8 +37,8 @@ describe('correlations', () => {
           index: 'apm-*',
           serviceName: 'actualServiceName',
           transactionName: 'actualTransactionName',
-          start: '01-01-2021',
-          end: '31-01-2021',
+          start: '2020',
+          end: '2021',
           environment: 'dev',
           percentileThresholdValue: 75,
         },
@@ -33,22 +46,17 @@ describe('correlations', () => {
       expect(query).toEqual({
         bool: {
           filter: [
-            { term: { 'processor.event': 'transaction' } },
-            {
-              term: {
-                'service.name': 'actualServiceName',
-              },
-            },
             {
               term: {
-                'transaction.name': 'actualTransactionName',
+                'processor.event': 'transaction',
               },
             },
             {
               range: {
                 '@timestamp': {
-                  gte: '01-01-2021',
-                  lte: '31-01-2021',
+                  format: 'epoch_millis',
+                  gte: 1577836800000,
+                  lte: 1609459200000,
                 },
               },
             },
@@ -57,6 +65,16 @@ describe('correlations', () => {
                 'service.environment': 'dev',
               },
             },
+            {
+              term: {
+                'service.name': 'actualServiceName',
+              },
+            },
+            {
+              term: {
+                'transaction.name': 'actualTransactionName',
+              },
+            },
             {
               range: {
                 'transaction.duration.us': {
@@ -71,7 +89,7 @@ describe('correlations', () => {
 
     it('returns a query considering a custom field/value pair', () => {
       const query = getQueryWithParams({
-        params: { index: 'apm-*' },
+        params: { index: 'apm-*', start: '2020', end: '2021' },
         fieldName: 'actualFieldName',
         fieldValue: 'actualFieldValue',
       });
@@ -79,6 +97,15 @@ describe('correlations', () => {
         bool: {
           filter: [
             { term: { 'processor.event': 'transaction' } },
+            {
+              range: {
+                '@timestamp': {
+                  format: 'epoch_millis',
+                  gte: 1577836800000,
+                  lte: 1609459200000,
+                },
+              },
+            },
             {
               term: {
                 actualFieldName: 'actualFieldValue',
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts
index 08ba4b23fec35b..e0ddfc1b053b5f 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts
@@ -5,16 +5,19 @@
  * 2.0.
  */
 
+import { pipe } from 'fp-ts/lib/pipeable';
+import { getOrElse } from 'fp-ts/lib/Either';
+import { failure } from 'io-ts/lib/PathReporter';
+import * as t from 'io-ts';
+
 import type { estypes } from '@elastic/elasticsearch';
-import {
-  PROCESSOR_EVENT,
-  SERVICE_NAME,
-  TRANSACTION_DURATION,
-  TRANSACTION_NAME,
-} from '../../../../common/elasticsearch_fieldnames';
+import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
 import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
-import { environmentQuery as getEnvironmentQuery } from '../../../utils/queries';
-import { ProcessorEvent } from '../../../../common/processor_event';
+import { rangeRt } from '../../../routes/default_api_types';
+
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+
+import { getCorrelationsFilters } from '../../correlations/get_filters';
 
 const getPercentileThresholdValueQuery = (
   percentileThresholdValue: number | undefined
@@ -39,26 +42,6 @@ export const getTermsQuery = (
   return fieldName && fieldValue ? [{ term: { [fieldName]: fieldValue } }] : [];
 };
 
-const getRangeQuery = (
-  start?: string,
-  end?: string
-): estypes.QueryDslQueryContainer[] => {
-  if (start === undefined && end === undefined) {
-    return [];
-  }
-
-  return [
-    {
-      range: {
-        '@timestamp': {
-          ...(start !== undefined ? { gte: start } : {}),
-          ...(end !== undefined ? { lte: end } : {}),
-        },
-      },
-    },
-  ];
-};
-
 interface QueryParams {
   params: SearchServiceParams;
   fieldName?: string;
@@ -71,21 +54,37 @@ export const getQueryWithParams = ({
 }: QueryParams) => {
   const {
     environment,
+    kuery,
     serviceName,
     start,
     end,
     percentileThresholdValue,
+    transactionType,
     transactionName,
   } = params;
+
+  // converts string based start/end to epochmillis
+  const setup = pipe(
+    rangeRt.decode({ start, end }),
+    getOrElse<t.Errors, { start: number; end: number }>((errors) => {
+      throw new Error(failure(errors).join('\n'));
+    })
+  ) as Setup & SetupTimeRange;
+
+  const filters = getCorrelationsFilters({
+    setup,
+    environment,
+    kuery,
+    serviceName,
+    transactionType,
+    transactionName,
+  });
+
   return {
     bool: {
       filter: [
-        ...getTermsQuery(PROCESSOR_EVENT, ProcessorEvent.transaction),
-        ...getTermsQuery(SERVICE_NAME, serviceName),
-        ...getTermsQuery(TRANSACTION_NAME, transactionName),
+        ...filters,
         ...getTermsQuery(fieldName, fieldValue),
-        ...getRangeQuery(start, end),
-        ...getEnvironmentQuery(environment),
         ...getPercentileThresholdValueQuery(percentileThresholdValue),
       ] as estypes.QueryDslQueryContainer[],
     },
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts
index 24741ebaa2daef..678328dce1a190 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts
@@ -15,7 +15,7 @@ import {
   BucketCorrelation,
 } from './query_correlation';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 const expectations = [1, 3, 5];
 const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }];
 const fractions = [1, 2, 4, 5];
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts
index 89bdd4280d3249..8929b31b3ecb17 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts
@@ -16,7 +16,7 @@ import {
   shouldBeExcluded,
 } from './query_field_candidates';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 
 describe('query_field_candidates', () => {
   describe('shouldBeExcluded', () => {
@@ -61,6 +61,15 @@ describe('query_field_candidates', () => {
                         'processor.event': 'transaction',
                       },
                     },
+                    {
+                      range: {
+                        '@timestamp': {
+                          format: 'epoch_millis',
+                          gte: 1577836800000,
+                          lte: 1609459200000,
+                        },
+                      },
+                    },
                   ],
                 },
               },
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts
index ea5a1f55bc9246..7ffbc5208e41e7 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts
@@ -16,7 +16,7 @@ import {
   getTermsAggRequest,
 } from './query_field_value_pairs';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 
 describe('query_field_value_pairs', () => {
   describe('getTermsAggRequest', () => {
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts
index 6052841d277c3d..3e7d4a52e4de2e 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts
@@ -14,7 +14,7 @@ import {
   getTransactionDurationRangesRequest,
 } from './query_fractions';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }];
 
 describe('query_fractions', () => {
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts
index 2be94463522605..ace91779479601 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts
@@ -14,7 +14,7 @@ import {
   getTransactionDurationHistogramRequest,
 } from './query_histogram';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 const interval = 100;
 
 describe('query_histogram', () => {
@@ -40,6 +40,15 @@ describe('query_histogram', () => {
                     'processor.event': 'transaction',
                   },
                 },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1577836800000,
+                      lte: 1609459200000,
+                    },
+                  },
+                },
               ],
             },
           },
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts
index 9ed529ccabddbe..ebd78f12485102 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts
@@ -14,7 +14,7 @@ import {
   getHistogramIntervalRequest,
 } from './query_histogram_interval';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 
 describe('query_histogram_interval', () => {
   describe('getHistogramIntervalRequest', () => {
@@ -43,6 +43,15 @@ describe('query_histogram_interval', () => {
                     'processor.event': 'transaction',
                   },
                 },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1577836800000,
+                      lte: 1609459200000,
+                    },
+                  },
+                },
               ],
             },
           },
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts
similarity index 77%
rename from x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts
rename to x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts
index bb366ea29fed48..76aab1cd979c94 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts
@@ -10,13 +10,13 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import {
-  fetchTransactionDurationHistogramRangesteps,
+  fetchTransactionDurationHistogramRangeSteps,
   getHistogramIntervalRequest,
-} from './query_histogram_rangesteps';
+} from './query_histogram_range_steps';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 
-describe('query_histogram_rangesteps', () => {
+describe('query_histogram_range_steps', () => {
   describe('getHistogramIntervalRequest', () => {
     it('returns the request body for the histogram interval request', () => {
       const req = getHistogramIntervalRequest(params);
@@ -43,6 +43,15 @@ describe('query_histogram_rangesteps', () => {
                     'processor.event': 'transaction',
                   },
                 },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1577836800000,
+                      lte: 1609459200000,
+                    },
+                  },
+                },
               ],
             },
           },
@@ -53,13 +62,14 @@ describe('query_histogram_rangesteps', () => {
     });
   });
 
-  describe('fetchTransactionDurationHistogramRangesteps', () => {
+  describe('fetchTransactionDurationHistogramRangeSteps', () => {
     it('fetches the range steps for the log histogram', async () => {
       const esClientSearchMock = jest.fn((req: estypes.SearchRequest): {
         body: estypes.SearchResponse;
       } => {
         return {
           body: ({
+            hits: { total: { value: 10 } },
             aggregations: {
               transaction_duration_max: {
                 value: 10000,
@@ -76,7 +86,7 @@ describe('query_histogram_rangesteps', () => {
         search: esClientSearchMock,
       } as unknown) as ElasticsearchClient;
 
-      const resp = await fetchTransactionDurationHistogramRangesteps(
+      const resp = await fetchTransactionDurationHistogramRangeSteps(
         esClientMock,
         params
       );
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts
similarity index 83%
rename from x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts
rename to x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts
index e537165ca53f37..6ee5dd6bcdf83b 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts
@@ -23,6 +23,14 @@ import type { SearchServiceParams } from '../../../../common/search_strategies/c
 
 import { getQueryWithParams } from './get_query_with_params';
 
+const getHistogramRangeSteps = (min: number, max: number, steps: number) => {
+  // A d3 based scale function as a helper to get equally distributed bins on a log scale.
+  const logFn = scaleLog().domain([min, max]).range([1, steps]);
+  return [...Array(steps).keys()]
+    .map(logFn.invert)
+    .map((d) => (isNaN(d) ? 0 : d));
+};
+
 export const getHistogramIntervalRequest = (
   params: SearchServiceParams
 ): estypes.SearchRequest => ({
@@ -37,19 +45,24 @@ export const getHistogramIntervalRequest = (
   },
 });
 
-export const fetchTransactionDurationHistogramRangesteps = async (
+export const fetchTransactionDurationHistogramRangeSteps = async (
   esClient: ElasticsearchClient,
   params: SearchServiceParams
 ): Promise<number[]> => {
+  const steps = 100;
+
   const resp = await esClient.search(getHistogramIntervalRequest(params));
 
+  if ((resp.body.hits.total as estypes.SearchTotalHits).value === 0) {
+    return getHistogramRangeSteps(0, 1, 100);
+  }
+
   if (resp.body.aggregations === undefined) {
     throw new Error(
-      'fetchTransactionDurationHistogramInterval failed, did not return aggregations.'
+      'fetchTransactionDurationHistogramRangeSteps failed, did not return aggregations.'
     );
   }
 
-  const steps = 100;
   const min = (resp.body.aggregations
     .transaction_duration_min as estypes.AggregationsValueAggregate).value;
   const max =
@@ -57,9 +70,5 @@ export const fetchTransactionDurationHistogramRangesteps = async (
       .transaction_duration_max as estypes.AggregationsValueAggregate).value *
     2;
 
-  // A d3 based scale function as a helper to get equally distributed bins on a log scale.
-  const logFn = scaleLog().domain([min, max]).range([1, steps]);
-  return [...Array(steps).keys()]
-    .map(logFn.invert)
-    .map((d) => (isNaN(d) ? 0 : d));
+  return getHistogramRangeSteps(min, max, steps);
 };
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts
index 0c319aee0fb2b7..f0d01a4849f9fd 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts
@@ -14,7 +14,7 @@ import {
   getTransactionDurationPercentilesRequest,
 } from './query_percentiles';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 
 describe('query_percentiles', () => {
   describe('getTransactionDurationPercentilesRequest', () => {
@@ -41,10 +41,20 @@ describe('query_percentiles', () => {
                     'processor.event': 'transaction',
                   },
                 },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1577836800000,
+                      lte: 1609459200000,
+                    },
+                  },
+                },
               ],
             },
           },
           size: 0,
+          track_total_hits: true,
         },
         index: params.index,
       });
@@ -53,6 +63,7 @@ describe('query_percentiles', () => {
 
   describe('fetchTransactionDurationPercentiles', () => {
     it('fetches the percentiles', async () => {
+      const totalDocs = 10;
       const percentilesValues = {
         '1.0': 5.0,
         '5.0': 25.0,
@@ -68,6 +79,7 @@ describe('query_percentiles', () => {
       } => {
         return {
           body: ({
+            hits: { total: { value: totalDocs } },
             aggregations: {
               transaction_duration_percentiles: {
                 values: percentilesValues,
@@ -86,7 +98,7 @@ describe('query_percentiles', () => {
         params
       );
 
-      expect(resp).toEqual(percentilesValues);
+      expect(resp).toEqual({ percentiles: percentilesValues, totalDocs });
       expect(esClientSearchMock).toHaveBeenCalledTimes(1);
     });
   });
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts
index 18dcefb59a11a5..c80f5d836c0ef1 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts
@@ -38,6 +38,7 @@ export const getTransactionDurationPercentilesRequest = (
   return {
     index: params.index,
     body: {
+      track_total_hits: true,
       query,
       size: 0,
       aggs: {
@@ -61,7 +62,7 @@ export const fetchTransactionDurationPercentiles = async (
   percents?: number[],
   fieldName?: string,
   fieldValue?: string
-): Promise<Record<string, number>> => {
+): Promise<{ totalDocs: number; percentiles: Record<string, number> }> => {
   const resp = await esClient.search<ResponseHit>(
     getTransactionDurationPercentilesRequest(
       params,
@@ -71,14 +72,22 @@ export const fetchTransactionDurationPercentiles = async (
     )
   );
 
+  // return early with no results if the search didn't return any documents
+  if ((resp.body.hits.total as estypes.SearchTotalHits).value === 0) {
+    return { totalDocs: 0, percentiles: {} };
+  }
+
   if (resp.body.aggregations === undefined) {
     throw new Error(
       'fetchTransactionDurationPercentiles failed, did not return aggregations.'
     );
   }
-  return (
-    (resp.body.aggregations
-      .transaction_duration_percentiles as estypes.AggregationsTDigestPercentilesAggregate)
-      .values ?? {}
-  );
+
+  return {
+    totalDocs: (resp.body.hits.total as estypes.SearchTotalHits).value,
+    percentiles:
+      (resp.body.aggregations
+        .transaction_duration_percentiles as estypes.AggregationsTDigestPercentilesAggregate)
+        .values ?? {},
+  };
 };
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts
index 9451928e47ded3..7d18efc360563f 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts
@@ -14,7 +14,7 @@ import {
   getTransactionDurationRangesRequest,
 } from './query_ranges';
 
-const params = { index: 'apm-*' };
+const params = { index: 'apm-*', start: '2020', end: '2021' };
 const rangeSteps = [1, 3, 5];
 
 describe('query_ranges', () => {
@@ -59,6 +59,15 @@ describe('query_ranges', () => {
                     'processor.event': 'transaction',
                   },
                 },
+                {
+                  range: {
+                    '@timestamp': {
+                      format: 'epoch_millis',
+                      gte: 1577836800000,
+                      lte: 1609459200000,
+                    },
+                  },
+                },
               ],
             },
           },
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts
index 6d4bfcdde99943..09775cb2eb0347 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts
@@ -122,6 +122,8 @@ describe('APM Correlations search strategy', () => {
       } as unknown) as SearchStrategyDependencies;
       params = {
         index: 'apm-*',
+        start: '2020',
+        end: '2021',
       };
     });
 
@@ -154,10 +156,22 @@ describe('APM Correlations search strategy', () => {
               },
               query: {
                 bool: {
-                  filter: [{ term: { 'processor.event': 'transaction' } }],
+                  filter: [
+                    { term: { 'processor.event': 'transaction' } },
+                    {
+                      range: {
+                        '@timestamp': {
+                          format: 'epoch_millis',
+                          gte: 1577836800000,
+                          lte: 1609459200000,
+                        },
+                      },
+                    },
+                  ],
                 },
               },
               size: 0,
+              track_total_hits: true,
             })
           );
         });
@@ -167,11 +181,17 @@ describe('APM Correlations search strategy', () => {
         it('retrieves the current request', async () => {
           const searchStrategy = await apmCorrelationsSearchStrategyProvider();
           const response = await searchStrategy
-            .search({ id: 'my-search-id', params }, {}, mockDeps)
+            .search({ params }, {}, mockDeps)
             .toPromise();
 
-          expect(response).toEqual(
-            expect.objectContaining({ id: 'my-search-id' })
+          const searchStrategyId = response.id;
+
+          const response2 = await searchStrategy
+            .search({ id: searchStrategyId, params }, {}, mockDeps)
+            .toPromise();
+
+          expect(response2).toEqual(
+            expect.objectContaining({ id: searchStrategyId })
           );
         });
       });
@@ -226,7 +246,7 @@ describe('APM Correlations search strategy', () => {
 
         expect(response2.id).toEqual(response1.id);
         expect(response2).toEqual(
-          expect.objectContaining({ loaded: 10, isRunning: false })
+          expect.objectContaining({ loaded: 100, isRunning: false })
         );
       });
     });
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts
index d6b4e0e7094b35..8f2e6913c0d062 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts
@@ -41,14 +41,40 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
         throw new Error('Invalid request parameters.');
       }
 
-      const id = request.id ?? uuid();
+      // The function to fetch the current state of the async search service.
+      // This will be either an existing service for a follow up fetch or a new one for new requests.
+      let getAsyncSearchServiceState: ReturnType<
+        typeof asyncSearchServiceProvider
+      >;
+
+      // If the request includes an ID, we require that the async search service already exists
+      // otherwise we throw an error. The client should never poll a service that's been cancelled or finished.
+      // This also avoids instantiating async search services when the service gets called with random IDs.
+      if (typeof request.id === 'string') {
+        const existingGetAsyncSearchServiceState = asyncSearchServiceMap.get(
+          request.id
+        );
 
-      const getAsyncSearchServiceState =
-        asyncSearchServiceMap.get(id) ??
-        asyncSearchServiceProvider(deps.esClient.asCurrentUser, request.params);
+        if (typeof existingGetAsyncSearchServiceState === 'undefined') {
+          throw new Error(
+            `AsyncSearchService with ID '${request.id}' does not exist.`
+          );
+        }
+
+        getAsyncSearchServiceState = existingGetAsyncSearchServiceState;
+      } else {
+        getAsyncSearchServiceState = asyncSearchServiceProvider(
+          deps.esClient.asCurrentUser,
+          request.params
+        );
+      }
+
+      // Reuse the request's id or create a new one.
+      const id = request.id ?? uuid();
 
       const {
         error,
+        log,
         isRunning,
         loaded,
         started,
@@ -76,6 +102,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
         isRunning,
         isPartial: isRunning,
         rawResponse: {
+          log,
           took,
           values,
           percentileThresholdValue,
diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts
index 63de0a59d4894a..4313ad58ecbc09 100644
--- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts
+++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts
@@ -14,6 +14,7 @@ describe('aggregation utils', () => {
       expect(expectations).toEqual([0, 0.5, 1]);
       expect(ranges).toEqual([{ to: 0 }, { from: 0, to: 1 }, { from: 1 }]);
     });
+
     it('returns expectations and ranges based on given percentiles #2', async () => {
       const { expectations, ranges } = computeExpectationsAndRanges([1, 3, 5]);
       expect(expectations).toEqual([1, 2, 4, 5]);
@@ -24,6 +25,7 @@ describe('aggregation utils', () => {
         { from: 5 },
       ]);
     });
+
     it('returns expectations and ranges with adjusted fractions', async () => {
       const { expectations, ranges } = computeExpectationsAndRanges([
         1,
@@ -45,5 +47,97 @@ describe('aggregation utils', () => {
         { from: 5 },
       ]);
     });
+
+    // TODO identify these results derived from the array of percentiles are usable with the ES correlation aggregation
+    it('returns expectation and ranges adjusted when percentiles have equal values', async () => {
+      const { expectations, ranges } = computeExpectationsAndRanges([
+        5000,
+        5000,
+        3090428,
+        3090428,
+        3090428,
+        3618812,
+        3618812,
+        3618812,
+        3618812,
+        3696636,
+        3696636,
+        3696636,
+        3696636,
+        3696636,
+        3696636,
+      ]);
+      expect(expectations).toEqual([
+        5000,
+        1856256.7999999998,
+        3392361.714285714,
+        3665506.4,
+        3696636,
+      ]);
+      expect(ranges).toEqual([
+        {
+          to: 5000,
+        },
+        {
+          from: 5000,
+          to: 5000,
+        },
+        {
+          from: 5000,
+          to: 3090428,
+        },
+        {
+          from: 3090428,
+          to: 3090428,
+        },
+        {
+          from: 3090428,
+          to: 3090428,
+        },
+        {
+          from: 3090428,
+          to: 3618812,
+        },
+        {
+          from: 3618812,
+          to: 3618812,
+        },
+        {
+          from: 3618812,
+          to: 3618812,
+        },
+        {
+          from: 3618812,
+          to: 3618812,
+        },
+        {
+          from: 3618812,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+          to: 3696636,
+        },
+        {
+          from: 3696636,
+        },
+      ]);
+    });
   });
 });
diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts
new file mode 100644
index 00000000000000..cc8f48fb589445
--- /dev/null
+++ b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts
@@ -0,0 +1,266 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import request from 'superagent';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { registry } from '../../common/registry';
+
+import { PartialSearchRequest } from '../../../../plugins/apm/server/lib/search_strategies/correlations/search_strategy';
+
+function parseBfetchResponse(resp: request.Response): Array<Record<string, any>> {
+  return resp.text
+    .trim()
+    .split('\n')
+    .map((item) => JSON.parse(item));
+}
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const retry = getService('retry');
+  const supertest = getService('supertest');
+
+  const getRequestBody = () => {
+    const partialSearchRequest: PartialSearchRequest = {
+      params: {
+        index: 'apm-*',
+        environment: 'ENVIRONMENT_ALL',
+        start: '2020',
+        end: '2021',
+        percentileThreshold: 95,
+      },
+    };
+
+    return {
+      batch: [
+        {
+          request: partialSearchRequest,
+          options: { strategy: 'apmCorrelationsSearchStrategy' },
+        },
+      ],
+    };
+  };
+
+  registry.when(
+    'correlations latency_ml overall without data',
+    { config: 'trial', archives: [] },
+    () => {
+      it('handles the empty state', async () => {
+        const intialResponse = await supertest
+          .post(`/internal/bsearch`)
+          .set('kbn-xsrf', 'foo')
+          .send(getRequestBody());
+
+        expect(intialResponse.status).to.eql(
+          200,
+          `Expected status to be '200', got '${intialResponse.status}'`
+        );
+        expect(intialResponse.body).to.eql(
+          {},
+          `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify(
+            intialResponse.body
+          )}'`
+        );
+
+        const body = parseBfetchResponse(intialResponse)[0];
+
+        expect(typeof body.result).to.be('object');
+        const { result } = body;
+
+        expect(typeof result?.id).to.be('string');
+
+        // pass on id for follow up queries
+        const searchStrategyId = result.id;
+
+        // follow up request body including search strategy ID
+        const reqBody = getRequestBody();
+        reqBody.batch[0].request.id = searchStrategyId;
+
+        let followUpResponse: Record<string, any> = {};
+
+        // continues querying until the search strategy finishes
+        await retry.waitForWithTimeout(
+          'search strategy eventually completes and returns full results',
+          5000,
+          async () => {
+            const response = await supertest
+              .post(`/internal/bsearch`)
+              .set('kbn-xsrf', 'foo')
+              .send(reqBody);
+
+            followUpResponse = parseBfetchResponse(response)[0];
+
+            return (
+              followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined
+            );
+          }
+        );
+
+        expect(followUpResponse?.error).to.eql(
+          undefined,
+          `search strategy should not return an error, got: ${JSON.stringify(
+            followUpResponse?.error
+          )}`
+        );
+
+        const followUpResult = followUpResponse.result;
+        expect(followUpResult?.isRunning).to.eql(false, 'search strategy should not be running');
+        expect(followUpResult?.isPartial).to.eql(
+          false,
+          'search strategy result should not be partial'
+        );
+        expect(followUpResult?.id).to.eql(
+          searchStrategyId,
+          'search strategy id should match original id'
+        );
+        expect(followUpResult?.isRestored).to.eql(
+          true,
+          'search strategy response should be restored'
+        );
+        expect(followUpResult?.loaded).to.eql(100, 'loaded state should be 100');
+        expect(followUpResult?.total).to.eql(100, 'total state should be 100');
+
+        expect(typeof followUpResult?.rawResponse).to.be('object');
+
+        const { rawResponse: finalRawResponse } = followUpResult;
+
+        expect(typeof finalRawResponse?.took).to.be('number');
+        expect(finalRawResponse?.percentileThresholdValue).to.be(undefined);
+        expect(finalRawResponse?.overallHistogram).to.be(undefined);
+        expect(finalRawResponse?.values.length).to.be(0);
+        expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([
+          'Fetched 95th percentile value of undefined based on 0 documents.',
+          'Abort service since percentileThresholdValue could not be determined.',
+        ]);
+      });
+    }
+  );
+
+  registry.when(
+    'Correlations latency_ml with data and opbeans-node args',
+    { config: 'trial', archives: ['ml_8.0.0'] },
+    () => {
+      // putting this into a single `it` because the responses depend on each other
+      it('queries the search strategy and returns results', async () => {
+        const intialResponse = await supertest
+          .post(`/internal/bsearch`)
+          .set('kbn-xsrf', 'foo')
+          .send(getRequestBody());
+
+        expect(intialResponse.status).to.eql(
+          200,
+          `Expected status to be '200', got '${intialResponse.status}'`
+        );
+        expect(intialResponse.body).to.eql(
+          {},
+          `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify(
+            intialResponse.body
+          )}'`
+        );
+
+        const body = parseBfetchResponse(intialResponse)[0];
+
+        expect(typeof body?.result).to.be('object');
+        const { result } = body;
+
+        expect(typeof result?.id).to.be('string');
+
+        // pass on id for follow up queries
+        const searchStrategyId = result.id;
+
+        expect(result?.loaded).to.be(0);
+        expect(result?.total).to.be(100);
+        expect(result?.isRunning).to.be(true);
+        expect(result?.isPartial).to.be(true);
+        expect(result?.isRestored).to.eql(
+          false,
+          `Expected response result to be not restored. Got: '${result?.isRestored}'`
+        );
+        expect(typeof result?.rawResponse).to.be('object');
+
+        const { rawResponse } = result;
+
+        expect(typeof rawResponse?.took).to.be('number');
+        expect(rawResponse?.values).to.eql([]);
+
+        // follow up request body including search strategy ID
+        const reqBody = getRequestBody();
+        reqBody.batch[0].request.id = searchStrategyId;
+
+        let followUpResponse: Record<string, any> = {};
+
+        // continues querying until the search strategy finishes
+        await retry.waitForWithTimeout(
+          'search strategy eventually completes and returns full results',
+          5000,
+          async () => {
+            const response = await supertest
+              .post(`/internal/bsearch`)
+              .set('kbn-xsrf', 'foo')
+              .send(reqBody);
+            followUpResponse = parseBfetchResponse(response)[0];
+
+            return (
+              followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined
+            );
+          }
+        );
+
+        expect(followUpResponse?.error).to.eql(
+          undefined,
+          `Finished search strategy should not return an error, got: ${JSON.stringify(
+            followUpResponse?.error
+          )}`
+        );
+
+        const followUpResult = followUpResponse.result;
+        expect(followUpResult?.isRunning).to.eql(
+          false,
+          `Expected finished result not to be running. Got: ${followUpResult?.isRunning}`
+        );
+        expect(followUpResult?.isPartial).to.eql(
+          false,
+          `Expected finished result not to be partial. Got: ${followUpResult?.isPartial}`
+        );
+        expect(followUpResult?.id).to.be(searchStrategyId);
+        expect(followUpResult?.isRestored).to.be(true);
+        expect(followUpResult?.loaded).to.be(100);
+        expect(followUpResult?.total).to.be(100);
+
+        expect(typeof followUpResult?.rawResponse).to.be('object');
+
+        const { rawResponse: finalRawResponse } = followUpResult;
+
+        expect(typeof finalRawResponse?.took).to.be('number');
+        expect(finalRawResponse?.percentileThresholdValue).to.be(1404927.875);
+        expect(finalRawResponse?.overallHistogram.length).to.be(101);
+
+        expect(finalRawResponse?.values.length).to.eql(
+          1,
+          `Expected 1 identified correlations, got ${finalRawResponse?.values.length}.`
+        );
+        expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([
+          'Fetched 95th percentile value of 1404927.875 based on 989 documents.',
+          'Loaded histogram range steps.',
+          'Loaded overall histogram chart data.',
+          'Loaded percentiles.',
+          'Identified 67 fieldCandidates.',
+          'Identified 339 fieldValuePairs.',
+          'Loaded fractions and totalDocCount of 989.',
+          'Identified 1 significant correlations out of 339 field/value pairs.',
+        ]);
+
+        const correlation = finalRawResponse?.values[0];
+        expect(typeof correlation).to.be('object');
+        expect(correlation?.field).to.be('transaction.result');
+        expect(correlation?.value).to.be('success');
+        expect(correlation?.correlation).to.be(0.37418510688551887);
+        expect(correlation?.ksTest).to.be(1.1238496968312214e-10);
+        expect(correlation?.histogram.length).to.be(101);
+      });
+    }
+  );
+}
diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts
index 813e0e4f3cdb89..a00fa1723fa3ec 100644
--- a/x-pack/test/apm_api_integration/tests/index.ts
+++ b/x-pack/test/apm_api_integration/tests/index.ts
@@ -32,6 +32,10 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte
       loadTestFile(require.resolve('./correlations/latency_slow_transactions'));
     });
 
+    describe('correlations/latency_ml', function () {
+      loadTestFile(require.resolve('./correlations/latency_ml'));
+    });
+
     describe('correlations/latency_overall', function () {
       loadTestFile(require.resolve('./correlations/latency_overall'));
     });

From 139326a06f0aeee3151a4667de9c90eb45af2569 Mon Sep 17 00:00:00 2001
From: Shahzad <shahzad.muhammad@elastic.co>
Date: Tue, 13 Jul 2021 23:31:34 +0200
Subject: [PATCH 10/27] [Exploratory view] Fixed bug in series storage hook
 (#105495)

---
 .../hooks/use_series_storage.test.tsx         | 131 ++++++++++++++++++
 .../hooks/use_series_storage.tsx              |   2 +-
 2 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.test.tsx

diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.test.tsx
new file mode 100644
index 00000000000000..c32acc47abd1b0
--- /dev/null
+++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.test.tsx
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useEffect } from 'react';
+
+import { UrlStorageContextProvider, useSeriesStorage } from './use_series_storage';
+import { render } from '@testing-library/react';
+
+const mockSingleSeries = {
+  'performance-distribution': {
+    reportType: 'data-distribution',
+    dataType: 'ux',
+    breakdown: 'user_agent.name',
+    time: { from: 'now-15m', to: 'now' },
+  },
+};
+
+const mockMultipleSeries = {
+  'performance-distribution': {
+    reportType: 'data-distribution',
+    dataType: 'ux',
+    breakdown: 'user_agent.name',
+    time: { from: 'now-15m', to: 'now' },
+  },
+  'kpi-over-time': {
+    reportType: 'kpi-over-time',
+    dataType: 'synthetics',
+    breakdown: 'user_agent.name',
+    time: { from: 'now-15m', to: 'now' },
+  },
+};
+
+describe('userSeries', function () {
+  function setupTestComponent(seriesData: any) {
+    const setData = jest.fn();
+    function TestComponent() {
+      const data = useSeriesStorage();
+
+      useEffect(() => {
+        setData(data);
+      }, [data]);
+
+      return <span>Test</span>;
+    }
+
+    render(
+      <UrlStorageContextProvider
+        storage={{ get: jest.fn().mockReturnValue(seriesData), set: jest.fn() }}
+      >
+        <TestComponent />
+      </UrlStorageContextProvider>
+    );
+
+    return setData;
+  }
+  it('should return expected result when there is one series', function () {
+    const setData = setupTestComponent(mockSingleSeries);
+
+    expect(setData).toHaveBeenCalledTimes(2);
+    expect(setData).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        allSeries: {
+          'performance-distribution': {
+            breakdown: 'user_agent.name',
+            dataType: 'ux',
+            reportType: 'data-distribution',
+            time: { from: 'now-15m', to: 'now' },
+          },
+        },
+        allSeriesIds: ['performance-distribution'],
+        firstSeries: {
+          breakdown: 'user_agent.name',
+          dataType: 'ux',
+          reportType: 'data-distribution',
+          time: { from: 'now-15m', to: 'now' },
+        },
+        firstSeriesId: 'performance-distribution',
+      })
+    );
+  });
+
+  it('should return expected result when there are multiple series series', function () {
+    const setData = setupTestComponent(mockMultipleSeries);
+
+    expect(setData).toHaveBeenCalledTimes(2);
+    expect(setData).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        allSeries: {
+          'performance-distribution': {
+            breakdown: 'user_agent.name',
+            dataType: 'ux',
+            reportType: 'data-distribution',
+            time: { from: 'now-15m', to: 'now' },
+          },
+          'kpi-over-time': {
+            reportType: 'kpi-over-time',
+            dataType: 'synthetics',
+            breakdown: 'user_agent.name',
+            time: { from: 'now-15m', to: 'now' },
+          },
+        },
+        allSeriesIds: ['performance-distribution', 'kpi-over-time'],
+        firstSeries: {
+          breakdown: 'user_agent.name',
+          dataType: 'ux',
+          reportType: 'data-distribution',
+          time: { from: 'now-15m', to: 'now' },
+        },
+        firstSeriesId: 'performance-distribution',
+      })
+    );
+  });
+
+  it('should return expected result when there are no series', function () {
+    const setData = setupTestComponent({});
+
+    expect(setData).toHaveBeenCalledTimes(2);
+    expect(setData).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        allSeries: {},
+        allSeriesIds: [],
+        firstSeries: undefined,
+        firstSeriesId: undefined,
+      })
+    );
+  });
+});
diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.tsx
index 0add5a19a95cc9..a47a124d14b4d9 100644
--- a/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.tsx
+++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_series_storage.tsx
@@ -67,7 +67,7 @@ export function UrlStorageContextProvider({
 
     setAllSeries(allSeriesN);
     setFirstSeriesId(allSeriesIds?.[0]);
-    setFirstSeries(allSeriesN?.[0]);
+    setFirstSeries(allSeriesN?.[allSeriesIds?.[0]]);
     (storage as IKbnUrlStateStorage).set(allSeriesKey, allShortSeries);
   }, [allShortSeries, storage]);
 

From 773b5b5d483dbe0dfffbd97c1ceaf7738130267c Mon Sep 17 00:00:00 2001
From: Frank Hassanabad <frank.hassanabad@elastic.co>
Date: Tue, 13 Jul 2021 15:42:16 -0600
Subject: [PATCH 11/27] [Security Solutions][Detection Engine] Removes EQL
 timestamp workaround and reduces test boiler plating (#105483)

## Summary

Removes EQL timestamp workaround we introduced earlier when we found the bug https://github.com/elastic/kibana/pull/103771 now that it has been fixed with the fields API https://github.com/elastic/elasticsearch/issues/74582

* Fixes the EQL timestamp issue by removing the workaround
* Introduces EQL timestamps being formatted as ISO8601 like we do with KQL
* Adds e2e tests for the EQL timestamps
* Removes some boiler plating around our e2e tests by adding two utilities of `getEqlRuleForSignalTesting` and `getThresholdRuleForSignalTesting` and reducing those e2e code areas.

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
---
 .../detection_engine/get_query_filter.test.ts |  10 +-
 .../detection_engine/get_query_filter.ts      |  18 +-
 .../tests/create_exceptions.ts                |  14 +-
 .../tests/generating_signals.ts               | 104 ++------
 .../tests/keyword_family/const_keyword.ts     |  18 +-
 .../tests/keyword_family/keyword.ts           |  16 +-
 .../keyword_mixed_with_const.ts               |  11 +-
 .../security_and_spaces/tests/timestamps.ts   | 251 +++++++++++-------
 .../detection_engine_api_integration/utils.ts |  42 +++
 9 files changed, 251 insertions(+), 233 deletions(-)

diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
index 7de082e778a07b..b38886296e74d4 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
@@ -1150,7 +1150,7 @@ describe('get_filter', () => {
             },
             {
               field: '@timestamp',
-              format: 'epoch_millis',
+              format: 'strict_date_optional_time',
             },
           ],
         },
@@ -1195,9 +1195,13 @@ describe('get_filter', () => {
               field: '*',
               include_unmapped: true,
             },
+            {
+              field: 'event.ingested',
+              format: 'strict_date_optional_time',
+            },
             {
               field: '@timestamp',
-              format: 'epoch_millis',
+              format: 'strict_date_optional_time',
             },
           ],
         },
@@ -1289,7 +1293,7 @@ describe('get_filter', () => {
             },
             {
               field: '@timestamp',
-              format: 'epoch_millis',
+              format: 'strict_date_optional_time',
             },
           ],
         },
diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
index 86e66577abd456..1e7bcb0002dad0 100644
--- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
+++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
@@ -79,6 +79,15 @@ export const buildEqlSearchRequest = (
   eventCategoryOverride: string | undefined
 ): EqlSearchRequest => {
   const timestamp = timestampOverride ?? '@timestamp';
+
+  const defaultTimeFields = ['@timestamp'];
+  const timestamps =
+    timestampOverride != null ? [timestampOverride, ...defaultTimeFields] : defaultTimeFields;
+  const docFields = timestamps.map((tstamp) => ({
+    field: tstamp,
+    format: 'strict_date_optional_time',
+  }));
+
   // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
   // allowing us to make 1024-item chunks of exception list items.
   // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
@@ -126,14 +135,7 @@ export const buildEqlSearchRequest = (
           field: '*',
           include_unmapped: true,
         },
-        {
-          field: '@timestamp',
-          // BUG: We have to format @timestamp until this bug is fixed with epoch_millis
-          // https://github.com/elastic/elasticsearch/issues/74582
-          // TODO: Remove epoch and use the same techniques from x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts
-          // where we format both the timestamp and any overrides as ISO8601
-          format: 'epoch_millis',
-        },
+        ...docFields,
       ],
     },
   };
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts
index 05b097cc87b616..e2251b0b8af084 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts
@@ -47,9 +47,10 @@ import {
   getSignalsByIds,
   findImmutableRuleById,
   getPrePackagedRulesStatus,
-  getRuleForSignalTesting,
   getOpenSignals,
   createRuleWithExceptionEntries,
+  getEqlRuleForSignalTesting,
+  getThresholdRuleForSignalTesting,
 } from '../../utils';
 import { ROLES } from '../../../../plugins/security_solution/common/test';
 import { createUserAndRole, deleteUserAndRole } from '../roles_users_utils';
@@ -615,10 +616,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('generates no signals when an exception is added for an EQL rule', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"',
           };
           const createdRule = await createRuleWithExceptionEntries(supertest, rule, [
@@ -637,11 +635,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('generates no signals when an exception is added for a threshold rule', async () => {
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'threshold-rule',
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 700,
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
index a1a97ac8bfd354..4972b485be06c7 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
@@ -21,11 +21,13 @@ import {
   createSignalsIndex,
   deleteAllAlerts,
   deleteSignalsIndex,
+  getEqlRuleForSignalTesting,
   getOpenSignals,
   getRuleForSignalTesting,
   getSignalsByIds,
   getSignalsByRuleIds,
   getSimpleRule,
+  getThresholdRuleForSignalTesting,
   waitForRuleSuccessOrStatus,
   waitForSignalsToBePresent,
 } from '../../utils';
@@ -273,16 +275,13 @@ export default ({ getService }: FtrProviderContext) => {
       describe('EQL Rules', () => {
         it('generates a correctly formatted signal from EQL non-sequence queries', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"',
           };
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
-          const signals = await getSignalsByRuleIds(supertest, ['eql-rule']);
+          const signals = await getSignalsByIds(supertest, [id]);
           expect(signals.hits.hits.length).eql(1);
           const fullSignal = signals.hits.hits[0]._source;
 
@@ -393,13 +392,7 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('generates up to max_signals for non-sequence EQL queries', async () => {
-          const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
-            query: 'any where true',
-          };
+          const rule: EqlCreateSchema = getEqlRuleForSignalTesting(['auditbeat-*']);
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 100, [id]);
@@ -412,17 +405,14 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('uses the provided event_category_override', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'config_change where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"',
             event_category_override: 'auditd.message_type',
           };
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
-          const signals = await getSignalsByRuleIds(supertest, ['eql-rule']);
+          const signals = await getSignalsByIds(supertest, [id]);
           expect(signals.hits.hits.length).eql(1);
           const fullSignal = signals.hits.hits[0]._source;
 
@@ -534,16 +524,13 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('generates building block signals from EQL sequences in the expected form', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'sequence by host.name [anomoly where true] [any where true]',
           };
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signals = await getSignalsByRuleIds(supertest, ['eql-rule']);
+          const signals = await getSignalsByIds(supertest, [id]);
           const buildingBlock = signals.hits.hits.find(
             (signal) =>
               signal._source.signal.depth === 1 &&
@@ -699,16 +686,13 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('generates shell signals from EQL sequences in the expected form', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'sequence by host.name [anomoly where true] [any where true]',
           };
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signalsOpen = await getSignalsByRuleIds(supertest, ['eql-rule']);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
           const sequenceSignal = signalsOpen.hits.hits.find(
             (signal) => signal._source.signal.depth === 2
           );
@@ -802,10 +786,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         it('generates up to max_signals with an EQL rule', async () => {
           const rule: EqlCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: 'eql-rule',
-            type: 'eql',
-            language: 'eql',
+            ...getEqlRuleForSignalTesting(['auditbeat-*']),
             query: 'sequence by host.name [any where true] [any where true]',
           };
           const { id } = await createRule(supertest, rule);
@@ -829,13 +810,8 @@ export default ({ getService }: FtrProviderContext) => {
 
       describe('Threshold Rules', () => {
         it('generates 1 signal from Threshold rules when threshold is met', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 700,
@@ -844,7 +820,7 @@ export default ({ getService }: FtrProviderContext) => {
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
-          const signalsOpen = await getSignalsByRuleIds(supertest, [ruleId]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
           expect(signalsOpen.hits.hits.length).eql(1);
           const fullSignal = signalsOpen.hits.hits[0]._source;
           const eventIds = fullSignal.signal.parents.map((event) => event.id);
@@ -895,13 +871,8 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('generates 2 signals from Threshold rules when threshold is met', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 100,
@@ -910,17 +881,13 @@ export default ({ getService }: FtrProviderContext) => {
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 2, [id]);
-          const signalsOpen = await getSignalsByRuleIds(supertest, [ruleId]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
           expect(signalsOpen.hits.hits.length).eql(2);
         });
 
         it('applies the provided query before bucketing ', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             query: 'host.id:"2ab45fc1c41e4c84bbd02202a7e5761f"',
             threshold: {
               field: 'process.name',
@@ -930,18 +897,13 @@ export default ({ getService }: FtrProviderContext) => {
           const { id } = await createRule(supertest, rule);
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
-          const signalsOpen = await getSignalsByRuleIds(supertest, [ruleId]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
           expect(signalsOpen.hits.hits.length).eql(1);
         });
 
         it('generates no signals from Threshold rules when threshold is met and cardinality is not met', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 100,
@@ -959,13 +921,8 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('generates no signals from Threshold rules when cardinality is met and threshold is not met', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 1000,
@@ -983,13 +940,8 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('generates signals from Threshold rules when threshold and cardinality are both met', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: 'host.id',
               value: 100,
@@ -1059,13 +1011,8 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('should not generate signals if only one field meets the threshold requirement', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: ['host.id', 'process.name'],
               value: 22,
@@ -1077,13 +1024,8 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('generates signals from Threshold rules when bucketing by multiple fields', async () => {
-          const ruleId = 'threshold-rule';
           const rule: ThresholdCreateSchema = {
-            ...getRuleForSignalTesting(['auditbeat-*']),
-            rule_id: ruleId,
-            type: 'threshold',
-            language: 'kuery',
-            query: '*:*',
+            ...getThresholdRuleForSignalTesting(['auditbeat-*']),
             threshold: {
               field: ['host.id', 'process.name', 'event.module'],
               value: 21,
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts
index b793fc635843e4..7d1a4d01fe27c4 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts
@@ -17,8 +17,10 @@ import {
   createSignalsIndex,
   deleteAllAlerts,
   deleteSignalsIndex,
+  getEqlRuleForSignalTesting,
   getRuleForSignalTesting,
   getSignalsById,
+  getThresholdRuleForSignalTesting,
   waitForRuleSuccessOrStatus,
   waitForSignalsToBePresent,
 } from '../../../utils';
@@ -84,10 +86,7 @@ export default ({ getService }: FtrProviderContext) => {
     describe('"eql" rule type', () => {
       it('should detect the "dataset_name_1" from "event.dataset" and have 4 signals', async () => {
         const rule: EqlCreateSchema = {
-          ...getRuleForSignalTesting(['const_keyword']),
-          rule_id: 'eql-rule',
-          type: 'eql',
-          language: 'eql',
+          ...getEqlRuleForSignalTesting(['const_keyword']),
           query: 'any where event.dataset=="dataset_name_1"',
         };
 
@@ -100,10 +99,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('should copy the "dataset_name_1" from "event.dataset"', async () => {
         const rule: EqlCreateSchema = {
-          ...getRuleForSignalTesting(['const_keyword']),
-          rule_id: 'eql-rule',
-          type: 'eql',
-          language: 'eql',
+          ...getEqlRuleForSignalTesting(['const_keyword']),
           query: 'any where event.dataset=="dataset_name_1"',
         };
 
@@ -126,11 +122,7 @@ export default ({ getService }: FtrProviderContext) => {
     describe('"threshold" rule type', async () => {
       it('should detect the "dataset_name_1" from "event.dataset"', async () => {
         const rule: ThresholdCreateSchema = {
-          ...getRuleForSignalTesting(['const_keyword']),
-          rule_id: 'threshold-rule',
-          type: 'threshold',
-          language: 'kuery',
-          query: '*:*',
+          ...getThresholdRuleForSignalTesting(['const_keyword']),
           threshold: {
             field: 'event.dataset',
             value: 1,
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword.ts
index d2d2898587ee2a..fba13c95c66acc 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword.ts
@@ -13,13 +13,16 @@ import {
   createSignalsIndex,
   deleteAllAlerts,
   deleteSignalsIndex,
+  getEqlRuleForSignalTesting,
   getRuleForSignalTesting,
   getSignalsById,
+  getThresholdRuleForSignalTesting,
   waitForRuleSuccessOrStatus,
   waitForSignalsToBePresent,
 } from '../../../utils';
 import {
   EqlCreateSchema,
+  QueryCreateSchema,
   ThresholdCreateSchema,
 } from '../../../../../plugins/security_solution/common/detection_engine/schemas/request';
 
@@ -47,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('"kql" rule type', () => {
       it('should detect the "dataset_name_1" from "event.dataset"', async () => {
-        const rule = {
+        const rule: QueryCreateSchema = {
           ...getRuleForSignalTesting(['keyword']),
           query: 'event.dataset: "dataset_name_1"',
         };
@@ -70,10 +73,7 @@ export default ({ getService }: FtrProviderContext) => {
     describe('"eql" rule type', () => {
       it('should detect the "dataset_name_1" from "event.dataset"', async () => {
         const rule: EqlCreateSchema = {
-          ...getRuleForSignalTesting(['keyword']),
-          rule_id: 'eql-rule',
-          type: 'eql',
-          language: 'eql',
+          ...getEqlRuleForSignalTesting(['keyword']),
           query: 'any where event.dataset=="dataset_name_1"',
         };
 
@@ -96,11 +96,7 @@ export default ({ getService }: FtrProviderContext) => {
     describe('"threshold" rule type', async () => {
       it('should detect the "dataset_name_1" from "event.dataset"', async () => {
         const rule: ThresholdCreateSchema = {
-          ...getRuleForSignalTesting(['keyword']),
-          rule_id: 'threshold-rule',
-          type: 'threshold',
-          language: 'kuery',
-          query: '*:*',
+          ...getThresholdRuleForSignalTesting(['keyword']),
           threshold: {
             field: 'event.dataset',
             value: 1,
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts
index 2ce88da13afab2..2a354a83a10aee 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts
@@ -17,6 +17,7 @@ import {
   createSignalsIndex,
   deleteAllAlerts,
   deleteSignalsIndex,
+  getEqlRuleForSignalTesting,
   getRuleForSignalTesting,
   getSignalsById,
   waitForRuleSuccessOrStatus,
@@ -90,10 +91,7 @@ export default ({ getService }: FtrProviderContext) => {
     describe('"eql" rule type', () => {
       it('should detect the "dataset_name_1" from "event.dataset" and have 8 signals, 4 from each index', async () => {
         const rule: EqlCreateSchema = {
-          ...getRuleForSignalTesting(['keyword', 'const_keyword']),
-          rule_id: 'eql-rule',
-          type: 'eql',
-          language: 'eql',
+          ...getEqlRuleForSignalTesting(['keyword', 'const_keyword']),
           query: 'any where event.dataset=="dataset_name_1"',
         };
 
@@ -106,10 +104,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('should copy the "dataset_name_1" from "event.dataset"', async () => {
         const rule: EqlCreateSchema = {
-          ...getRuleForSignalTesting(['keyword', 'const_keyword']),
-          rule_id: 'eql-rule',
-          type: 'eql',
-          language: 'eql',
+          ...getEqlRuleForSignalTesting(['keyword', 'const_keyword']),
           query: 'any where event.dataset=="dataset_name_1"',
         };
 
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts
index 8645fec287b074..2c304803ded897 100644
--- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts
+++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts
@@ -7,7 +7,10 @@
 
 import expect from '@kbn/expect';
 import { orderBy } from 'lodash';
-import { QueryCreateSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request';
+import {
+  EqlCreateSchema,
+  QueryCreateSchema,
+} from '../../../../plugins/security_solution/common/detection_engine/schemas/request';
 
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 import {
@@ -19,6 +22,7 @@ import {
   waitForSignalsToBePresent,
   getRuleForSignalTesting,
   getSignalsByIds,
+  getEqlRuleForSignalTesting,
 } from '../../utils';
 
 // eslint-disable-next-line import/no-default-export
@@ -54,27 +58,54 @@ export default ({ getService }: FtrProviderContext) => {
         );
       });
 
-      it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => {
-        const rule = getRuleForSignalTesting(['timestamp_in_seconds']);
-        const { id } = await createRule(supertest, rule);
-        await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
-        const signalsOpen = await getSignalsByIds(supertest, [id]);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
-        expect(hits).to.eql(['2021-06-02T23:33:15.000Z']);
+      describe('KQL query', () => {
+        it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => {
+          const rule = getRuleForSignalTesting(['timestamp_in_seconds']);
+          const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
+          expect(hits).to.eql(['2021-06-02T23:33:15.000Z']);
+        });
+
+        it('should still use the @timestamp field even with an override field. It should never use the override field', async () => {
+          const rule: QueryCreateSchema = {
+            ...getRuleForSignalTesting(['myfakeindex-5']),
+            timestamp_override: 'event.ingested',
+          };
+          const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
+          expect(hits).to.eql(['2020-12-16T15:16:18.000Z']);
+        });
       });
 
-      it('should still use the @timestamp field even with an override field. It should never use the override field', async () => {
-        const rule: QueryCreateSchema = {
-          ...getRuleForSignalTesting(['myfakeindex-5']),
-          timestamp_override: 'event.ingested',
-        };
-        const { id } = await createRule(supertest, rule);
-        await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
-        const signalsOpen = await getSignalsByIds(supertest, [id]);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
-        expect(hits).to.eql(['2020-12-16T15:16:18.000Z']);
+      describe('EQL query', () => {
+        it('should convert the @timestamp which is epoch_seconds into the correct ISO format for EQL', async () => {
+          const rule = getEqlRuleForSignalTesting(['timestamp_in_seconds']);
+          const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
+          expect(hits).to.eql(['2021-06-02T23:33:15.000Z']);
+        });
+
+        it('should still use the @timestamp field even with an override field. It should never use the override field', async () => {
+          const rule: EqlCreateSchema = {
+            ...getEqlRuleForSignalTesting(['myfakeindex-5']),
+            timestamp_override: 'event.ingested',
+          };
+          const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsByIds(supertest, [id]);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
+          expect(hits).to.eql(['2020-12-16T15:16:18.000Z']);
+        });
       });
     });
 
@@ -119,73 +150,91 @@ export default ({ getService }: FtrProviderContext) => {
         );
       });
 
-      it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => {
-        const rule: QueryCreateSchema = {
-          ...getRuleForSignalTesting(['myfa*']),
-          timestamp_override: 'event.ingested',
-        };
+      describe('KQL', () => {
+        it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => {
+          const rule: QueryCreateSchema = {
+            ...getRuleForSignalTesting(['myfa*']),
+            timestamp_override: 'event.ingested',
+          };
 
-        const { id } = await createRule(supertest, rule);
+          const { id } = await createRule(supertest, rule);
 
-        await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
-        await waitForSignalsToBePresent(supertest, 3, [id]);
-        const signalsResponse = await getSignalsByIds(supertest, [id], 3);
-        const signals = signalsResponse.hits.hits.map((hit) => hit._source);
-        const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
+          await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
+          await waitForSignalsToBePresent(supertest, 3, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id], 3);
+          const signals = signalsResponse.hits.hits.map((hit) => hit._source);
+          const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
 
-        expect(signalsOrderedByEventId.length).equal(3);
-      });
+          expect(signalsOrderedByEventId.length).equal(3);
+        });
 
-      it('should generate 2 signals with @timestamp', async () => {
-        const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']);
+        it('should generate 2 signals with @timestamp', async () => {
+          const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']);
 
-        const { id } = await createRule(supertest, rule);
+          const { id } = await createRule(supertest, rule);
 
-        await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
-        await waitForSignalsToBePresent(supertest, 2, [id]);
-        const signalsResponse = await getSignalsByIds(supertest, [id]);
-        const signals = signalsResponse.hits.hits.map((hit) => hit._source);
-        const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
+          await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
+          await waitForSignalsToBePresent(supertest, 2, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id]);
+          const signals = signalsResponse.hits.hits.map((hit) => hit._source);
+          const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
 
-        expect(signalsOrderedByEventId.length).equal(2);
-      });
+          expect(signalsOrderedByEventId.length).equal(2);
+        });
+
+        it('should generate 2 signals when timestamp override does not exist', async () => {
+          const rule: QueryCreateSchema = {
+            ...getRuleForSignalTesting(['myfa*']),
+            timestamp_override: 'event.fakeingestfield',
+          };
+          const { id } = await createRule(supertest, rule);
 
-      it('should generate 2 signals when timestamp override does not exist', async () => {
-        const rule: QueryCreateSchema = {
-          ...getRuleForSignalTesting(['myfa*']),
-          timestamp_override: 'event.fakeingestfield',
-        };
-        const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
+          await waitForSignalsToBePresent(supertest, 2, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id, id]);
+          const signals = signalsResponse.hits.hits.map((hit) => hit._source);
+          const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
 
-        await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
-        await waitForSignalsToBePresent(supertest, 2, [id]);
-        const signalsResponse = await getSignalsByIds(supertest, [id, id]);
-        const signals = signalsResponse.hits.hits.map((hit) => hit._source);
-        const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
+          expect(signalsOrderedByEventId.length).equal(2);
+        });
 
-        expect(signalsOrderedByEventId.length).equal(2);
+        /**
+         * We should not use the timestamp override as the "original_time" as that can cause
+         * confusion if you have both a timestamp and an override in the source event. Instead the "original_time"
+         * field should only be overridden by the "timestamp" since when we generate a signal
+         * and we add a new timestamp to the signal.
+         */
+        it('should NOT use the timestamp override as the "original_time"', async () => {
+          const rule: QueryCreateSchema = {
+            ...getRuleForSignalTesting(['myfakeindex-2']),
+            timestamp_override: 'event.ingested',
+          };
+          const { id } = await createRule(supertest, rule);
+
+          await waitForRuleSuccessOrStatus(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id, id]);
+          const hits = signalsResponse.hits.hits
+            .map((hit) => hit._source.signal.original_time)
+            .sort();
+          expect(hits).to.eql([undefined]);
+        });
       });
 
-      /**
-       * We should not use the timestamp override as the "original_time" as that can cause
-       * confusion if you have both a timestamp and an override in the source event. Instead the "original_time"
-       * field should only be overridden by the "timestamp" since when we generate a signal
-       * and we add a new timestamp to the signal.
-       */
-      it('should NOT use the timestamp override as the "original_time"', async () => {
-        const rule: QueryCreateSchema = {
-          ...getRuleForSignalTesting(['myfakeindex-2']),
-          timestamp_override: 'event.ingested',
-        };
-        const { id } = await createRule(supertest, rule);
-
-        await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
-        const signalsResponse = await getSignalsByIds(supertest, [id, id]);
-        const hits = signalsResponse.hits.hits
-          .map((hit) => hit._source.signal.original_time)
-          .sort();
-        expect(hits).to.eql([undefined]);
+      describe('EQL', () => {
+        it('should generate 2 signals with @timestamp', async () => {
+          const rule: EqlCreateSchema = getEqlRuleForSignalTesting(['myfa*']);
+
+          const { id } = await createRule(supertest, rule);
+
+          await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
+          await waitForSignalsToBePresent(supertest, 2, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id]);
+          const signals = signalsResponse.hits.hits.map((hit) => hit._source);
+          const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
+
+          expect(signalsOrderedByEventId.length).equal(2);
+        });
       });
     });
 
@@ -201,31 +250,33 @@ export default ({ getService }: FtrProviderContext) => {
         await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts');
       });
 
-      /**
-       * This represents our worst case scenario where this field is not mapped on any index
-       * We want to check that our logic continues to function within the constraints of search after
-       * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields
-       * Javascript does not support numbers this large, but without passing in a number of this size
-       * The search_after will continue to return the same results and not iterate to the next set
-       * So to circumvent this limitation of javascript we return the stringified version of Java's
-       * Long.MAX_VALUE so that search_after does not enter into an infinite loop.
-       *
-       * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
-       */
-      it('should generate 200 signals when timestamp override does not exist', async () => {
-        const rule: QueryCreateSchema = {
-          ...getRuleForSignalTesting(['auditbeat-*']),
-          timestamp_override: 'event.fakeingested',
-          max_signals: 200,
-        };
-
-        const { id } = await createRule(supertest, rule);
-        await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
-        await waitForSignalsToBePresent(supertest, 200, [id]);
-        const signalsResponse = await getSignalsByIds(supertest, [id], 200);
-        const signals = signalsResponse.hits.hits.map((hit) => hit._source);
-
-        expect(signals.length).equal(200);
+      describe('KQL', () => {
+        /**
+         * This represents our worst case scenario where this field is not mapped on any index
+         * We want to check that our logic continues to function within the constraints of search after
+         * Elasticsearch returns java's long.MAX_VALUE for unmapped date fields
+         * Javascript does not support numbers this large, but without passing in a number of this size
+         * The search_after will continue to return the same results and not iterate to the next set
+         * So to circumvent this limitation of javascript we return the stringified version of Java's
+         * Long.MAX_VALUE so that search_after does not enter into an infinite loop.
+         *
+         * ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
+         */
+        it('should generate 200 signals when timestamp override does not exist', async () => {
+          const rule: QueryCreateSchema = {
+            ...getRuleForSignalTesting(['auditbeat-*']),
+            timestamp_override: 'event.fakeingested',
+            max_signals: 200,
+          };
+
+          const { id } = await createRule(supertest, rule);
+          await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
+          await waitForSignalsToBePresent(supertest, 200, [id]);
+          const signalsResponse = await getSignalsByIds(supertest, [id], 200);
+          const signals = signalsResponse.hits.hits.map((hit) => hit._source);
+
+          expect(signals.length).equal(200);
+        });
       });
     });
   });
diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts
index 54252b19fc940f..ac11dd31c15e80 100644
--- a/x-pack/test/detection_engine_api_integration/utils.ts
+++ b/x-pack/test/detection_engine_api_integration/utils.ts
@@ -27,6 +27,8 @@ import {
   UpdateRulesSchema,
   FullResponseSchema,
   QueryCreateSchema,
+  EqlCreateSchema,
+  ThresholdCreateSchema,
 } from '../../plugins/security_solution/common/detection_engine/schemas/request';
 import { Signal } from '../../plugins/security_solution/server/lib/detection_engine/signals/types';
 import { signalsMigrationType } from '../../plugins/security_solution/server/lib/detection_engine/migrations/saved_objects';
@@ -123,6 +125,46 @@ export const getRuleForSignalTesting = (
   from: '1900-01-01T00:00:00.000Z',
 });
 
+/**
+ * This is a typical signal testing rule that is easy for most basic testing of output of EQL signals.
+ * It starts out in an enabled true state. The from is set very far back to test the basics of signal
+ * creation for EQL and testing by getting all the signals at once.
+ * @param ruleId The optional ruleId which is eql-rule by default.
+ * @param enabled Enables the rule on creation or not. Defaulted to true.
+ */
+export const getEqlRuleForSignalTesting = (
+  index: string[],
+  ruleId = 'eql-rule',
+  enabled = true
+): EqlCreateSchema => ({
+  ...getRuleForSignalTesting(index, ruleId, enabled),
+  type: 'eql',
+  language: 'eql',
+  query: 'any where true',
+});
+
+/**
+ * This is a typical signal testing rule that is easy for most basic testing of output of Threshold signals.
+ * It starts out in an enabled true state. The from is set very far back to test the basics of signal
+ * creation for Threshold and testing by getting all the signals at once.
+ * @param ruleId The optional ruleId which is threshold-rule by default.
+ * @param enabled Enables the rule on creation or not. Defaulted to true.
+ */
+export const getThresholdRuleForSignalTesting = (
+  index: string[],
+  ruleId = 'threshold-rule',
+  enabled = true
+): ThresholdCreateSchema => ({
+  ...getRuleForSignalTesting(index, ruleId, enabled),
+  type: 'threshold',
+  language: 'kuery',
+  query: '*:*',
+  threshold: {
+    field: 'process.name',
+    value: 21,
+  },
+});
+
 export const getRuleForSignalTestingWithTimestampOverride = (
   index: string[],
   ruleId = 'rule-1',

From 8f1dde56759b51583a9eb343306df41df0f795ac Mon Sep 17 00:00:00 2001
From: Kyle Pollich <kyle.pollich@elastic.co>
Date: Tue, 13 Jul 2021 17:48:30 -0400
Subject: [PATCH 12/27] Handle deprecated `$yml` code block language in
 integration READMEs (#105498)

---
 .../detail/overview/markdown_renderers.tsx    | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/markdown_renderers.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/markdown_renderers.tsx
index cbc2f7b5f78881..2cbdfe3671c4e6 100644
--- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/markdown_renderers.tsx
+++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/markdown_renderers.tsx
@@ -24,6 +24,12 @@ const REL_NOFOLLOW = 'nofollow';
 /** prevents the browser from sending the current address as referrer via the Referer HTTP header */
 const REL_NOREFERRER = 'noreferrer';
 
+// Maps deprecated code block languages to supported ones in prism.js
+const CODE_LANGUAGE_OVERRIDES: Record<string, string> = {
+  $json: 'json',
+  $yml: 'yml',
+};
+
 export const markdownRenderers = {
   root: ({ children }: { children: React.ReactNode[] }) => (
     <EuiText grow={true}>{children}</EuiText>
@@ -60,8 +66,17 @@ export const markdownRenderers = {
     </EuiLink>
   ),
   code: ({ language, value }: { language: string; value: string }) => {
-    // Old packages are using `$json`, which is not valid any more with the move to prism.js
-    const parsedLang = language === '$json' ? 'json' : language;
+    let parsedLang = language;
+
+    // Some integrations export code block content that includes language tags that have since
+    // been removed or deprecated in `prism.js`, the upstream depedency that handles syntax highlighting
+    // in EuiCodeBlock components
+    const languageOverride = CODE_LANGUAGE_OVERRIDES[language];
+
+    if (languageOverride) {
+      parsedLang = languageOverride;
+    }
+
     return (
       <EuiCodeBlock language={parsedLang} isCopyable>
         {value}

From c633dbef7b8f66ad971564c660220ffd616dba61 Mon Sep 17 00:00:00 2001
From: Justin Kambic <justin.kambic@elastic.co>
Date: Tue, 13 Jul 2021 18:05:55 -0400
Subject: [PATCH 13/27] [Uptime] Exclude all documents missing hash from TLS
 query (#105492)

* Exclude all documents missing `tls.server.hash.sha256` from TLS page query.

* Simplify query.
---
 x-pack/plugins/uptime/server/lib/requests/get_certs.test.ts | 2 +-
 x-pack/plugins/uptime/server/lib/requests/get_certs.ts      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/uptime/server/lib/requests/get_certs.test.ts b/x-pack/plugins/uptime/server/lib/requests/get_certs.test.ts
index 333824df174a65..6e06dea0436dbf 100644
--- a/x-pack/plugins/uptime/server/lib/requests/get_certs.test.ts
+++ b/x-pack/plugins/uptime/server/lib/requests/get_certs.test.ts
@@ -173,7 +173,7 @@ describe('getCerts', () => {
                   "filter": Array [
                     Object {
                       "exists": Object {
-                        "field": "tls.server",
+                        "field": "tls.server.hash.sha256",
                       },
                     },
                     Object {
diff --git a/x-pack/plugins/uptime/server/lib/requests/get_certs.ts b/x-pack/plugins/uptime/server/lib/requests/get_certs.ts
index 7639484f517374..86a9825f8a4856 100644
--- a/x-pack/plugins/uptime/server/lib/requests/get_certs.ts
+++ b/x-pack/plugins/uptime/server/lib/requests/get_certs.ts
@@ -64,7 +64,7 @@ export const getCerts: UMElasticsearchQueryFn<GetCertsParams, CertResult> = asyn
         filter: [
           {
             exists: {
-              field: 'tls.server',
+              field: 'tls.server.hash.sha256',
             },
           },
           {

From ef06cf7ec081107be9787c3a4c966dfe2fe69c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= <patryk.kopycinski@elastic.co>
Date: Wed, 14 Jul 2021 01:10:08 +0300
Subject: [PATCH 14/27] [Osquery] Revert fix Saved Query mapping (#105503)

---
 .../server/lib/saved_query/saved_object_mappings.ts  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
index 5535d707cf5c0a..537b6d7874ab8a 100644
--- a/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
+++ b/x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
@@ -24,7 +24,7 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'date',
     },
     created_by: {
-      type: 'keyword',
+      type: 'text',
     },
     platform: {
       type: 'keyword',
@@ -36,7 +36,7 @@ export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'date',
     },
     updated_by: {
-      type: 'keyword',
+      type: 'text',
     },
     interval: {
       type: 'keyword',
@@ -57,19 +57,19 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = {
       type: 'text',
     },
     name: {
-      type: 'keyword',
+      type: 'text',
     },
     created_at: {
       type: 'date',
     },
     created_by: {
-      type: 'keyword',
+      type: 'text',
     },
     updated_at: {
       type: 'date',
     },
     updated_by: {
-      type: 'keyword',
+      type: 'text',
     },
     queries: {
       properties: {
@@ -77,7 +77,7 @@ export const packSavedObjectMappings: SavedObjectsType['mappings'] = {
           type: 'keyword',
         },
         interval: {
-          type: 'keyword',
+          type: 'text',
         },
       },
     },

From bdf1069e564b8e1229632b6a01d6122d1b563e70 Mon Sep 17 00:00:00 2001
From: Frank Hassanabad <frank.hassanabad@elastic.co>
Date: Tue, 13 Jul 2021 16:23:58 -0600
Subject: [PATCH 15/27] [Security Solutions][Detection Engine] Removes dead
 duplicated code and marks other duplicated code (#105374)

## Summary

* Removes dead duplicated code from `security_solution` and  `lists`
* Adds notes and TODO's where we still have duplicated logic
* Adds notes where I saw that the original deviated from the copy from modifications in one file but not the other.
* DOES NOT fix the bugs existing in one copy but not the other. That should be done when the copied chunks are collapsed into a package. Instead see this issue where I marked those areas: https://github.com/elastic/kibana/issues/105378

See these two files where things have deviated from our duplications as an example:
[security_solution/public/common/components/autocomplete/field.tsx](https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx
)
[lists/public/exceptions/components/autocomplete/field.tsx](https://github.com/elastic/kibana/blob/master/x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx)

Ref PR where fixes are applied to one of the files but not the other (could be other PR's in addition to this one):
https://github.com/elastic/kibana/pull/87004

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
---
 .../src/autocomplete_operators/index.ts       |   3 +-
 .../src/autocomplete_operators/types.ts       |  19 --
 .../src/helpers/index.ts                      |   2 +-
 .../src/types/index.ts                        |   7 +-
 .../components/autocomplete/field.tsx         |   7 +
 .../autocomplete/field_value_match.tsx        |   5 +
 .../components/autocomplete/helpers.ts        |  10 +-
 .../hooks/use_field_value_autocomplete.ts     |   3 +
 .../components/autocomplete/operator.tsx      |   3 +-
 .../components/autocomplete/types.ts          |  11 -
 .../components/builder/entry_renderer.tsx     |   2 +-
 .../components/builder/helpers.test.ts        |   2 +-
 .../common/components/autocomplete/field.tsx  |   7 +
 .../autocomplete/field_value_exists.test.tsx  |  23 --
 .../autocomplete/field_value_exists.tsx       |  33 ---
 .../autocomplete/field_value_lists.test.tsx   | 214 ----------------
 .../autocomplete/field_value_lists.tsx        | 121 ---------
 .../autocomplete/field_value_match.tsx        |   5 +
 .../field_value_match_any.test.tsx            | 238 ------------------
 .../autocomplete/field_value_match_any.tsx    | 221 ----------------
 .../components/autocomplete/helpers.test.ts   | 167 +-----------
 .../common/components/autocomplete/helpers.ts |  76 +-----
 .../hooks/use_field_value_autocomplete.ts     |   4 +
 .../components/autocomplete/operator.test.tsx | 225 -----------------
 .../components/autocomplete/operator.tsx      |  82 ------
 .../common/components/autocomplete/types.ts   |  12 -
 26 files changed, 63 insertions(+), 1439 deletions(-)
 delete mode 100644 packages/kbn-securitysolution-list-utils/src/autocomplete_operators/types.ts
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.test.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.test.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.test.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/operator.test.tsx
 delete mode 100644 x-pack/plugins/security_solution/public/common/components/autocomplete/operator.tsx

diff --git a/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/index.ts b/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/index.ts
index 967cebc360f61b..051c359dc46129 100644
--- a/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/index.ts
+++ b/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/index.ts
@@ -11,8 +11,7 @@ import {
   ListOperatorEnum as OperatorEnum,
   ListOperatorTypeEnum as OperatorTypeEnum,
 } from '@kbn/securitysolution-io-ts-list-types';
-
-import { OperatorOption } from './types';
+import { OperatorOption } from '../types';
 
 export const isOperator: OperatorOption = {
   message: i18n.translate('lists.exceptions.isOperatorLabel', {
diff --git a/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/types.ts b/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/types.ts
deleted file mode 100644
index 1be21bb62a7fee..00000000000000
--- a/packages/kbn-securitysolution-list-utils/src/autocomplete_operators/types.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import type {
-  ListOperatorEnum as OperatorEnum,
-  ListOperatorTypeEnum as OperatorTypeEnum,
-} from '@kbn/securitysolution-io-ts-list-types';
-
-export interface OperatorOption {
-  message: string;
-  value: string;
-  operator: OperatorEnum;
-  type: OperatorTypeEnum;
-}
diff --git a/packages/kbn-securitysolution-list-utils/src/helpers/index.ts b/packages/kbn-securitysolution-list-utils/src/helpers/index.ts
index d208624b69fc5e..38446b2a08ec0f 100644
--- a/packages/kbn-securitysolution-list-utils/src/helpers/index.ts
+++ b/packages/kbn-securitysolution-list-utils/src/helpers/index.ts
@@ -43,7 +43,6 @@ import {
   isOneOfOperator,
   isOperator,
 } from '../autocomplete_operators';
-import { OperatorOption } from '../autocomplete_operators/types';
 
 import {
   BuilderEntry,
@@ -52,6 +51,7 @@ import {
   EmptyNestedEntry,
   ExceptionsBuilderExceptionItem,
   FormattedBuilderEntry,
+  OperatorOption,
 } from '../types';
 
 export const isEntryNested = (item: BuilderEntry): item is EntryNested => {
diff --git a/packages/kbn-securitysolution-list-utils/src/types/index.ts b/packages/kbn-securitysolution-list-utils/src/types/index.ts
index faf68ca1579812..537ac06a49f34b 100644
--- a/packages/kbn-securitysolution-list-utils/src/types/index.ts
+++ b/packages/kbn-securitysolution-list-utils/src/types/index.ts
@@ -23,7 +23,12 @@ import {
   EXCEPTION_LIST_NAMESPACE_AGNOSTIC,
 } from '@kbn/securitysolution-list-constants';
 
-import type { OperatorOption } from '../autocomplete_operators/types';
+export interface OperatorOption {
+  message: string;
+  value: string;
+  operator: OperatorEnum;
+  type: OperatorTypeEnum;
+}
 
 /**
  * @deprecated Use the one from core once it is in its own package which will be from:
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx b/x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx
index b3a5e36f12e404..47527914e71ff0 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx
@@ -28,6 +28,13 @@ interface OperatorProps {
   selectedField: IFieldType | undefined;
 }
 
+/**
+ * There is a copy within:
+ * x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx
+ *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
+ * NOTE: This has deviated from the copy and will have to be reconciled.
+ */
 export const FieldComponent: React.FC<OperatorProps> = ({
   fieldInputWidth,
   fieldTypeFilter = [],
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/field_value_match.tsx b/x-pack/plugins/lists/public/exceptions/components/autocomplete/field_value_match.tsx
index c1776280842c69..8dbe8f223ae5bd 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/field_value_match.tsx
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/field_value_match.tsx
@@ -47,6 +47,11 @@ interface AutocompleteFieldMatchProps {
   onError?: (arg: boolean) => void;
 }
 
+/**
+ * There is a copy of this within:
+ * x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match.tsx
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
+ */
 export const AutocompleteFieldMatchComponent: React.FC<AutocompleteFieldMatchProps> = ({
   placeholder,
   rowLabel,
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts b/x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
index 965214815eedf6..975416e272227c 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
@@ -10,6 +10,7 @@ import { EuiComboBoxOptionOption } from '@elastic/eui';
 import type { ListSchema, Type } from '@kbn/securitysolution-io-ts-list-types';
 import {
   EXCEPTION_OPERATORS,
+  OperatorOption,
   doesNotExistOperator,
   existsOperator,
   isNotOperator,
@@ -18,7 +19,7 @@ import {
 
 import { IFieldType } from '../../../../../../../src/plugins/data/common';
 
-import { GetGenericComboBoxPropsReturn, OperatorOption } from './types';
+import { GetGenericComboBoxPropsReturn } from './types';
 import * as i18n from './translations';
 
 /**
@@ -72,6 +73,10 @@ export const checkEmptyValue = (
 
 /**
  * Very basic validation for values
+ * There is a copy within:
+ * x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts
+ *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  *
  * @param param the value being checked
  * @param field the selected field
@@ -109,7 +114,10 @@ export const paramIsValid = (
 
 /**
  * Determines the options, selected values and option labels for EUI combo box
+ * There is a copy within:
+ * x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts
  *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  * @param options options user can select from
  * @param selectedOptions user selection if any
  * @param getLabel helper function to know which property to use for labels
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/hooks/use_field_value_autocomplete.ts b/x-pack/plugins/lists/public/exceptions/components/autocomplete/hooks/use_field_value_autocomplete.ts
index 674bb5e5537d92..63d3925d6d64d3 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/hooks/use_field_value_autocomplete.ts
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/hooks/use_field_value_autocomplete.ts
@@ -33,7 +33,10 @@ export interface UseFieldValueAutocompleteProps {
 }
 /**
  * Hook for using the field value autocomplete service
+ * There is a copy within:
+ * x-pack/plugins/security_solution/public/common/components/autocomplete/hooks/use_field_value_autocomplete.ts
  *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  */
 export const useFieldValueAutocomplete = ({
   selectedField,
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/operator.tsx b/x-pack/plugins/lists/public/exceptions/components/autocomplete/operator.tsx
index 7fc221c5a097c3..0d2fe5bd664be1 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/operator.tsx
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/operator.tsx
@@ -7,11 +7,12 @@
 
 import React, { useCallback, useMemo } from 'react';
 import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
+import { OperatorOption } from '@kbn/securitysolution-list-utils';
 
 import { IFieldType } from '../../../../../../../src/plugins/data/common';
 
 import { getGenericComboBoxProps, getOperators } from './helpers';
-import { GetGenericComboBoxPropsReturn, OperatorOption } from './types';
+import { GetGenericComboBoxPropsReturn } from './types';
 
 const AS_PLAIN_TEXT = { asPlainText: true };
 
diff --git a/x-pack/plugins/lists/public/exceptions/components/autocomplete/types.ts b/x-pack/plugins/lists/public/exceptions/components/autocomplete/types.ts
index 76d5b7758007b4..07f1903fb70e1c 100644
--- a/x-pack/plugins/lists/public/exceptions/components/autocomplete/types.ts
+++ b/x-pack/plugins/lists/public/exceptions/components/autocomplete/types.ts
@@ -6,20 +6,9 @@
  */
 
 import { EuiComboBoxOptionOption } from '@elastic/eui';
-import type {
-  ListOperatorEnum as OperatorEnum,
-  ListOperatorTypeEnum as OperatorTypeEnum,
-} from '@kbn/securitysolution-io-ts-list-types';
 
 export interface GetGenericComboBoxPropsReturn {
   comboOptions: EuiComboBoxOptionOption[];
   labels: string[];
   selectedComboOptions: EuiComboBoxOptionOption[];
 }
-
-export interface OperatorOption {
-  message: string;
-  value: string;
-  operator: OperatorEnum;
-  type: OperatorTypeEnum;
-}
diff --git a/x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.tsx b/x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.tsx
index 7daef8467dd1a9..c54da89766d76e 100644
--- a/x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.tsx
+++ b/x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.tsx
@@ -18,6 +18,7 @@ import {
   BuilderEntry,
   EXCEPTION_OPERATORS_ONLY_LISTS,
   FormattedBuilderEntry,
+  OperatorOption,
   getEntryOnFieldChange,
   getEntryOnListChange,
   getEntryOnMatchAnyChange,
@@ -32,7 +33,6 @@ import { IFieldType, IIndexPattern } from '../../../../../../../src/plugins/data
 import { HttpStart } from '../../../../../../../src/core/public';
 import { FieldComponent } from '../autocomplete/field';
 import { OperatorComponent } from '../autocomplete/operator';
-import { OperatorOption } from '../autocomplete/types';
 import { AutocompleteFieldExistsComponent } from '../autocomplete/field_value_exists';
 import { AutocompleteFieldMatchComponent } from '../autocomplete/field_value_match';
 import { AutocompleteFieldMatchAnyComponent } from '../autocomplete/field_value_match_any';
diff --git a/x-pack/plugins/lists/public/exceptions/components/builder/helpers.test.ts b/x-pack/plugins/lists/public/exceptions/components/builder/helpers.test.ts
index 212db40f3168cc..afeac2d1bf4de3 100644
--- a/x-pack/plugins/lists/public/exceptions/components/builder/helpers.test.ts
+++ b/x-pack/plugins/lists/public/exceptions/components/builder/helpers.test.ts
@@ -24,6 +24,7 @@ import {
   EmptyEntry,
   ExceptionsBuilderExceptionItem,
   FormattedBuilderEntry,
+  OperatorOption,
   doesNotExistOperator,
   existsOperator,
   filterExceptionItems,
@@ -64,7 +65,6 @@ import { getEntryNestedMock } from '../../../../common/schemas/types/entry_neste
 import { getEntryMatchMock } from '../../../../common/schemas/types/entry_match.mock';
 import { getEntryMatchAnyMock } from '../../../../common/schemas/types/entry_match_any.mock';
 import { getListResponseMock } from '../../../../common/schemas/response/list_schema.mock';
-import { OperatorOption } from '../autocomplete/types';
 import { getEntryListMock } from '../../../../common/schemas/types/entry_list.mock';
 
 // TODO: ALL THESE TESTS SHOULD BE MOVED TO @kbn/securitysolution-list-utils for its helper. The only reason why they're here is due to missing other packages we hae to create or missing things from kbn packages such as mocks from kibana core
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx
index d4185fe639695e..a175a9b847c715 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/field.tsx
@@ -25,6 +25,13 @@ interface OperatorProps {
   onChange: (a: IFieldType[]) => void;
 }
 
+/**
+ * There is a copy within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/field.tsx
+ *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
+ * NOTE: This has deviated from the copy and will have to be reconciled.
+ */
 export const FieldComponent: React.FC<OperatorProps> = ({
   placeholder,
   selectedField,
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.test.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.test.tsx
deleted file mode 100644
index b6300581f12dd8..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.test.tsx
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-import { mount } from 'enzyme';
-
-import { AutocompleteFieldExistsComponent } from './field_value_exists';
-
-describe('AutocompleteFieldExistsComponent', () => {
-  test('it renders field disabled', () => {
-    const wrapper = mount(<AutocompleteFieldExistsComponent placeholder="Placeholder text" />);
-
-    expect(
-      wrapper
-        .find(`[data-test-subj="valuesAutocompleteComboBox existsComboxBox"] input`)
-        .prop('disabled')
-    ).toBeTruthy();
-  });
-});
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.tsx
deleted file mode 100644
index 715ba52701177b..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_exists.tsx
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-import { EuiFormRow, EuiComboBox } from '@elastic/eui';
-
-interface AutocompleteFieldExistsProps {
-  placeholder: string;
-  rowLabel?: string;
-}
-
-export const AutocompleteFieldExistsComponent: React.FC<AutocompleteFieldExistsProps> = ({
-  placeholder,
-  rowLabel,
-}): JSX.Element => (
-  <EuiFormRow label={rowLabel} fullWidth>
-    <EuiComboBox
-      placeholder={placeholder}
-      options={[]}
-      selectedOptions={[]}
-      onChange={undefined}
-      isDisabled
-      data-test-subj="valuesAutocompleteComboBox existsComboxBox"
-      fullWidth
-    />
-  </EuiFormRow>
-);
-
-AutocompleteFieldExistsComponent.displayName = 'AutocompleteFieldExists';
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.test.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.test.tsx
deleted file mode 100644
index 164b8e8d2a6d68..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.test.tsx
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-import { mount } from 'enzyme';
-import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
-import { waitFor } from '@testing-library/react';
-
-import { getField } from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks';
-import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
-import { getFoundListSchemaMock } from '../../../../../lists/common/schemas/response/found_list_schema.mock';
-import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
-import { DATE_NOW, VERSION, IMMUTABLE } from '../../../../../lists/common/constants.mock';
-
-import { AutocompleteFieldListsComponent } from './field_value_lists';
-
-jest.mock('../../../common/lib/kibana');
-const mockStart = jest.fn();
-const mockKeywordList: ListSchema = {
-  ...getListResponseMock(),
-  id: 'keyword_list',
-  type: 'keyword',
-  name: 'keyword list',
-};
-const mockResult = { ...getFoundListSchemaMock() };
-mockResult.data = [...mockResult.data, mockKeywordList];
-jest.mock('@kbn/securitysolution-list-hooks', () => {
-  const originalModule = jest.requireActual('@kbn/securitysolution-list-hooks');
-
-  return {
-    ...originalModule,
-    useFindLists: () => ({
-      loading: false,
-      start: mockStart.mockReturnValue(mockResult),
-      result: mockResult,
-      error: undefined,
-    }),
-  };
-});
-
-describe('AutocompleteFieldListsComponent', () => {
-  test('it renders disabled if "isDisabled" is true', async () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue="some-list-id"
-        isLoading={false}
-        isClearable={true}
-        isDisabled
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper
-        .find(`[data-test-subj="valuesAutocompleteComboBox listsComboxBox"] input`)
-        .prop('disabled')
-    ).toBeTruthy();
-  });
-
-  test('it renders loading if "isLoading" is true', async () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('@tags')}
-        selectedValue=""
-        isLoading
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    wrapper
-      .find(`[data-test-subj="valuesAutocompleteComboBox listsComboxBox"] button`)
-      .at(0)
-      .simulate('click');
-    expect(
-      wrapper
-        .find(
-          `EuiComboBoxOptionsList[data-test-subj="valuesAutocompleteComboBox listsComboxBox-optionsList"]`
-        )
-        .prop('isLoading')
-    ).toBeTruthy();
-  });
-
-  test('it allows user to clear values if "isClearable" is true', async () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue=""
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-    expect(
-      wrapper
-        .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
-        .prop('options')
-    ).toEqual([{ label: 'some name' }]);
-  });
-
-  test('it correctly displays lists that match the selected "keyword" field esType', () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('@tags')}
-        selectedValue=""
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    wrapper.find('[data-test-subj="comboBoxToggleListButton"] button').simulate('click');
-
-    expect(
-      wrapper
-        .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
-        .prop('options')
-    ).toEqual([{ label: 'keyword list' }]);
-  });
-
-  test('it correctly displays lists that match the selected "ip" field esType', () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue=""
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    wrapper.find('[data-test-subj="comboBoxToggleListButton"] button').simulate('click');
-
-    expect(
-      wrapper
-        .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
-        .prop('options')
-    ).toEqual([{ label: 'some name' }]);
-  });
-
-  test('it correctly displays selected list', async () => {
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue="some-list-id"
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper
-        .find(`[data-test-subj="valuesAutocompleteComboBox listsComboxBox"] EuiComboBoxPill`)
-        .at(0)
-        .text()
-    ).toEqual('some name');
-  });
-
-  test('it invokes "onChange" when option selected', async () => {
-    const mockOnChange = jest.fn();
-    const wrapper = mount(
-      <AutocompleteFieldListsComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue=""
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={mockOnChange}
-      />
-    );
-
-    ((wrapper.find(EuiComboBox).props() as unknown) as {
-      onChange: (a: EuiComboBoxOptionOption[]) => void;
-    }).onChange([{ label: 'some name' }]);
-
-    await waitFor(() => {
-      expect(mockOnChange).toHaveBeenCalledWith({
-        created_at: DATE_NOW,
-        created_by: 'some user',
-        description: 'some description',
-        id: 'some-list-id',
-        meta: {},
-        name: 'some name',
-        tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e',
-        type: 'ip',
-        updated_at: DATE_NOW,
-        updated_by: 'some user',
-        _version: undefined,
-        version: VERSION,
-        deserializer: undefined,
-        serializer: undefined,
-        immutable: IMMUTABLE,
-      });
-    });
-  });
-});
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx
deleted file mode 100644
index e8a3c2e70c75b2..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React, { useState, useEffect, useCallback, useMemo } from 'react';
-import { EuiFormRow, EuiComboBoxOptionOption, EuiComboBox } from '@elastic/eui';
-
-import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
-import { useFindLists } from '@kbn/securitysolution-list-hooks';
-import { IFieldType } from '../../../../../../../src/plugins/data/common';
-import { useKibana } from '../../../common/lib/kibana';
-import { filterFieldToList, getGenericComboBoxProps } from './helpers';
-import * as i18n from './translations';
-
-interface AutocompleteFieldListsProps {
-  placeholder: string;
-  selectedField: IFieldType | undefined;
-  selectedValue: string | undefined;
-  isLoading: boolean;
-  isDisabled: boolean;
-  isClearable: boolean;
-  isRequired?: boolean;
-  rowLabel?: string;
-  onChange: (arg: ListSchema) => void;
-}
-
-export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsProps> = ({
-  placeholder,
-  rowLabel,
-  selectedField,
-  selectedValue,
-  isLoading = false,
-  isDisabled = false,
-  isClearable = false,
-  isRequired = false,
-  onChange,
-}): JSX.Element => {
-  const [error, setError] = useState<string | undefined>(undefined);
-  const { http } = useKibana().services;
-  const [lists, setLists] = useState<ListSchema[]>([]);
-  const { loading, result, start } = useFindLists();
-  const getLabel = useCallback(({ name }) => name, []);
-
-  const optionsMemo = useMemo(() => filterFieldToList(lists, selectedField), [
-    lists,
-    selectedField,
-  ]);
-  const selectedOptionsMemo = useMemo(() => {
-    if (selectedValue != null) {
-      const list = lists.filter(({ id }) => id === selectedValue);
-      return list ?? [];
-    } else {
-      return [];
-    }
-  }, [selectedValue, lists]);
-  const { comboOptions, labels, selectedComboOptions } = useMemo(
-    () =>
-      getGenericComboBoxProps<ListSchema>({
-        options: optionsMemo,
-        selectedOptions: selectedOptionsMemo,
-        getLabel,
-      }),
-    [optionsMemo, selectedOptionsMemo, getLabel]
-  );
-
-  const handleValuesChange = useCallback(
-    (newOptions: EuiComboBoxOptionOption[]) => {
-      const [newValue] = newOptions.map(({ label }) => optionsMemo[labels.indexOf(label)]);
-      onChange(newValue ?? '');
-    },
-    [labels, optionsMemo, onChange]
-  );
-
-  const setIsTouchedValue = useCallback((): void => {
-    setError(selectedValue == null ? i18n.FIELD_REQUIRED_ERR : undefined);
-  }, [selectedValue]);
-
-  useEffect(() => {
-    if (result != null) {
-      setLists(result.data);
-    }
-  }, [result]);
-
-  useEffect(() => {
-    if (selectedField != null) {
-      start({
-        http,
-        pageIndex: 1,
-        pageSize: 500,
-      });
-    }
-  }, [selectedField, start, http]);
-
-  const isLoadingState = useMemo((): boolean => isLoading || loading, [isLoading, loading]);
-
-  return (
-    <EuiFormRow label={rowLabel} error={error} isInvalid={error != null} fullWidth>
-      <EuiComboBox
-        placeholder={placeholder}
-        isDisabled={isDisabled}
-        isLoading={isLoadingState}
-        isClearable={isClearable}
-        options={comboOptions}
-        selectedOptions={selectedComboOptions}
-        onChange={handleValuesChange}
-        isInvalid={error != null}
-        onBlur={setIsTouchedValue}
-        singleSelection={{ asPlainText: true }}
-        sortMatchesBy="startsWith"
-        data-test-subj="valuesAutocompleteComboBox listsComboxBox"
-        fullWidth
-        async
-      />
-    </EuiFormRow>
-  );
-};
-
-AutocompleteFieldListsComponent.displayName = 'AutocompleteFieldList';
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match.tsx
index 9cb219e7a8d456..21d1d9b4b31aa1 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match.tsx
@@ -38,6 +38,11 @@ interface AutocompleteFieldMatchProps {
   onError?: (arg: boolean) => void;
 }
 
+/**
+ * There is a copy of this within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/field_value_match.tsx
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
+ */
 export const AutocompleteFieldMatchComponent: React.FC<AutocompleteFieldMatchProps> = ({
   placeholder,
   rowLabel,
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.test.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.test.tsx
deleted file mode 100644
index 6b479c5ab8c4c9..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.test.tsx
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-import { mount, ReactWrapper } from 'enzyme';
-import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
-import { act } from '@testing-library/react';
-
-import {
-  fields,
-  getField,
-} from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks';
-import { AutocompleteFieldMatchAnyComponent } from './field_value_match_any';
-import { useFieldValueAutocomplete } from './hooks/use_field_value_autocomplete';
-
-jest.mock('./hooks/use_field_value_autocomplete');
-
-describe('AutocompleteFieldMatchAnyComponent', () => {
-  let wrapper: ReactWrapper;
-  const getValueSuggestionsMock = jest
-    .fn()
-    .mockResolvedValue([false, true, ['value 3', 'value 4'], jest.fn()]);
-
-  beforeEach(() => {
-    (useFieldValueAutocomplete as jest.Mock).mockReturnValue([
-      false,
-      true,
-      ['value 1', 'value 2'],
-      getValueSuggestionsMock,
-    ]);
-  });
-
-  afterEach(() => {
-    jest.clearAllMocks();
-    wrapper.unmount();
-  });
-
-  test('it renders disabled if "isDisabled" is true', () => {
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue={['126.45.211.34']}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={false}
-        isDisabled={true}
-        onChange={jest.fn()}
-        onError={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="valuesAutocompleteMatchAny"] input`).prop('disabled')
-    ).toBeTruthy();
-  });
-
-  test('it renders loading if "isLoading" is true', () => {
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue={[]}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={true}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-        onError={jest.fn()}
-      />
-    );
-    wrapper.find(`[data-test-subj="valuesAutocompleteMatchAny"] button`).at(0).simulate('click');
-    expect(
-      wrapper
-        .find(`EuiComboBoxOptionsList[data-test-subj="valuesAutocompleteMatchAny-optionsList"]`)
-        .prop('isLoading')
-    ).toBeTruthy();
-  });
-
-  test('it allows user to clear values if "isClearable" is true', () => {
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue={['126.45.211.34']}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={true}
-        isDisabled={false}
-        onChange={jest.fn()}
-        onError={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper
-        .find(`[data-test-subj="comboBoxInput"]`)
-        .hasClass('euiComboBox__inputWrap-isClearable')
-    ).toBeTruthy();
-  });
-
-  test('it correctly displays selected value', () => {
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue={['126.45.211.34']}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-        onError={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="valuesAutocompleteMatchAny"] EuiComboBoxPill`).at(0).text()
-    ).toEqual('126.45.211.34');
-  });
-
-  test('it invokes "onChange" when new value created', async () => {
-    const mockOnChange = jest.fn();
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('ip')}
-        selectedValue={[]}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={mockOnChange}
-        onError={jest.fn()}
-      />
-    );
-
-    ((wrapper.find(EuiComboBox).props() as unknown) as {
-      onCreateOption: (a: string) => void;
-    }).onCreateOption('126.45.211.34');
-
-    expect(mockOnChange).toHaveBeenCalledWith(['126.45.211.34']);
-  });
-
-  test('it invokes "onChange" when new value selected', async () => {
-    const mockOnChange = jest.fn();
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        selectedValue={[]}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={mockOnChange}
-        onError={jest.fn()}
-      />
-    );
-
-    ((wrapper.find(EuiComboBox).props() as unknown) as {
-      onChange: (a: EuiComboBoxOptionOption[]) => void;
-    }).onChange([{ label: 'value 1' }]);
-
-    expect(mockOnChange).toHaveBeenCalledWith(['value 1']);
-  });
-
-  test('it refreshes autocomplete with search query when new value searched', () => {
-    wrapper = mount(
-      <AutocompleteFieldMatchAnyComponent
-        rowLabel={'Row Label'}
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        selectedValue={[]}
-        indexPattern={{
-          id: '1234',
-          title: 'logstash-*',
-          fields,
-        }}
-        isLoading={false}
-        isClearable={false}
-        isDisabled={false}
-        onChange={jest.fn()}
-      />
-    );
-    act(() => {
-      ((wrapper.find(EuiComboBox).props() as unknown) as {
-        onSearchChange: (a: string) => void;
-      }).onSearchChange('value 1');
-    });
-
-    expect(useFieldValueAutocomplete).toHaveBeenCalledWith({
-      selectedField: getField('machine.os.raw'),
-      operatorType: 'match_any',
-      query: 'value 1',
-      fieldValue: [],
-      indexPattern: {
-        id: '1234',
-        title: 'logstash-*',
-        fields,
-      },
-    });
-  });
-});
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.tsx
deleted file mode 100644
index dbfdaf9749b6de..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_match_any.tsx
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React, { useState, useCallback, useMemo } from 'react';
-import { EuiFormRow, EuiComboBoxOptionOption, EuiComboBox } from '@elastic/eui';
-import { uniq } from 'lodash';
-
-import { ListOperatorTypeEnum as OperatorTypeEnum } from '@kbn/securitysolution-io-ts-list-types';
-import { IFieldType, IIndexPattern } from '../../../../../../../src/plugins/data/common';
-import { useFieldValueAutocomplete } from './hooks/use_field_value_autocomplete';
-import { getGenericComboBoxProps, paramIsValid } from './helpers';
-import { GetGenericComboBoxPropsReturn } from './types';
-
-import * as i18n from './translations';
-
-interface AutocompleteFieldMatchAnyProps {
-  placeholder: string;
-  selectedField: IFieldType | undefined;
-  selectedValue: string[];
-  indexPattern: IIndexPattern | undefined;
-  isLoading: boolean;
-  isDisabled: boolean;
-  isClearable: boolean;
-  isRequired?: boolean;
-  rowLabel?: string;
-  onChange: (arg: string[]) => void;
-  onError?: (arg: boolean) => void;
-}
-
-export const AutocompleteFieldMatchAnyComponent: React.FC<AutocompleteFieldMatchAnyProps> = ({
-  placeholder,
-  rowLabel,
-  selectedField,
-  selectedValue,
-  indexPattern,
-  isLoading,
-  isDisabled = false,
-  isClearable = false,
-  isRequired = false,
-  onChange,
-  onError,
-}): JSX.Element => {
-  const [searchQuery, setSearchQuery] = useState('');
-  const [touched, setIsTouched] = useState(false);
-  const [error, setError] = useState<string | undefined>(undefined);
-  const [isLoadingSuggestions, isSuggestingValues, suggestions] = useFieldValueAutocomplete({
-    selectedField,
-    operatorType: OperatorTypeEnum.MATCH_ANY,
-    fieldValue: selectedValue,
-    query: searchQuery,
-    indexPattern,
-  });
-  const getLabel = useCallback((option: string): string => option, []);
-  const optionsMemo = useMemo(
-    (): string[] => (selectedValue ? uniq([...selectedValue, ...suggestions]) : suggestions),
-    [suggestions, selectedValue]
-  );
-  const { comboOptions, labels, selectedComboOptions } = useMemo(
-    (): GetGenericComboBoxPropsReturn =>
-      getGenericComboBoxProps<string>({
-        options: optionsMemo,
-        selectedOptions: selectedValue,
-        getLabel,
-      }),
-    [optionsMemo, selectedValue, getLabel]
-  );
-
-  const handleError = useCallback(
-    (err: string | undefined): void => {
-      setError((existingErr): string | undefined => {
-        const oldErr = existingErr != null;
-        const newErr = err != null;
-        if (oldErr !== newErr && onError != null) {
-          onError(newErr);
-        }
-
-        return err;
-      });
-    },
-    [setError, onError]
-  );
-
-  const handleValuesChange = useCallback(
-    (newOptions: EuiComboBoxOptionOption[]): void => {
-      const newValues: string[] = newOptions.map(({ label }) => optionsMemo[labels.indexOf(label)]);
-      handleError(undefined);
-      onChange(newValues);
-    },
-    [handleError, labels, onChange, optionsMemo]
-  );
-
-  const handleSearchChange = useCallback(
-    (searchVal: string) => {
-      if (searchVal === '') {
-        handleError(undefined);
-      }
-
-      if (searchVal !== '' && selectedField != null) {
-        const err = paramIsValid(searchVal, selectedField, isRequired, touched);
-        handleError(err);
-
-        setSearchQuery(searchVal);
-      }
-    },
-    [handleError, isRequired, selectedField, touched]
-  );
-
-  const handleCreateOption = useCallback(
-    (option: string): boolean | void => {
-      const err = paramIsValid(option, selectedField, isRequired, touched);
-      handleError(err);
-
-      if (err != null) {
-        // Explicitly reject the user's input
-        return false;
-      } else {
-        onChange([...(selectedValue || []), option]);
-      }
-    },
-    [handleError, isRequired, onChange, selectedField, selectedValue, touched]
-  );
-
-  const setIsTouchedValue = useCallback((): void => {
-    handleError(selectedComboOptions.length === 0 ? i18n.FIELD_REQUIRED_ERR : undefined);
-    setIsTouched(true);
-  }, [setIsTouched, handleError, selectedComboOptions]);
-
-  const inputPlaceholder = useMemo(
-    (): string => (isLoading || isLoadingSuggestions ? i18n.LOADING : placeholder),
-    [isLoading, isLoadingSuggestions, placeholder]
-  );
-
-  const isLoadingState = useMemo((): boolean => isLoading || isLoadingSuggestions, [
-    isLoading,
-    isLoadingSuggestions,
-  ]);
-
-  const defaultInput = useMemo((): JSX.Element => {
-    return (
-      <EuiFormRow
-        label={rowLabel}
-        error={error}
-        isInvalid={selectedField != null && error != null}
-        fullWidth
-      >
-        <EuiComboBox
-          placeholder={inputPlaceholder}
-          isLoading={isLoadingState}
-          isClearable={isClearable}
-          isDisabled={isDisabled}
-          options={comboOptions}
-          selectedOptions={selectedComboOptions}
-          onChange={handleValuesChange}
-          onSearchChange={handleSearchChange}
-          onCreateOption={handleCreateOption}
-          isInvalid={selectedField != null && error != null}
-          onBlur={setIsTouchedValue}
-          delimiter=", "
-          data-test-subj="valuesAutocompleteMatchAny"
-          fullWidth
-          async
-        />
-      </EuiFormRow>
-    );
-  }, [
-    comboOptions,
-    error,
-    handleCreateOption,
-    handleSearchChange,
-    handleValuesChange,
-    inputPlaceholder,
-    isClearable,
-    isDisabled,
-    isLoadingState,
-    rowLabel,
-    selectedComboOptions,
-    selectedField,
-    setIsTouchedValue,
-  ]);
-
-  if (!isSuggestingValues && selectedField != null) {
-    switch (selectedField.type) {
-      case 'number':
-        return (
-          <EuiFormRow
-            label={rowLabel}
-            error={error}
-            isInvalid={selectedField != null && error != null}
-            fullWidth
-          >
-            <EuiComboBox
-              noSuggestions
-              placeholder={inputPlaceholder}
-              isLoading={isLoadingState}
-              isClearable={isClearable}
-              isDisabled={isDisabled}
-              selectedOptions={selectedComboOptions}
-              onChange={handleValuesChange}
-              onSearchChange={handleSearchChange}
-              onCreateOption={handleCreateOption}
-              isInvalid={selectedField != null && error != null}
-              onFocus={setIsTouchedValue}
-              delimiter=", "
-              data-test-subj="valuesAutocompleteMatchAnyNumber"
-              fullWidth
-            />
-          </EuiFormRow>
-        );
-      default:
-        return defaultInput;
-    }
-  }
-
-  return defaultInput;
-};
-
-AutocompleteFieldMatchAnyComponent.displayName = 'AutocompleteFieldMatchAny';
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts
index ae695bf7be9782..1618de245365dc 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts
@@ -8,65 +8,13 @@
 import moment from 'moment';
 import '../../../common/mock/match_media';
 import { getField } from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks';
-import { IFieldType } from '../../../../../../../src/plugins/data/common';
 
 import * as i18n from './translations';
-import {
-  EXCEPTION_OPERATORS,
-  isOperator,
-  isNotOperator,
-  existsOperator,
-  doesNotExistOperator,
-} from '@kbn/securitysolution-list-utils';
-import {
-  getOperators,
-  checkEmptyValue,
-  paramIsValid,
-  getGenericComboBoxProps,
-  typeMatch,
-  filterFieldToList,
-} from './helpers';
-import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
-import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
+import { checkEmptyValue, paramIsValid, getGenericComboBoxProps } from './helpers';
 
 describe('helpers', () => {
   // @ts-ignore
   moment.suppressDeprecationWarnings = true;
-  describe('#getOperators', () => {
-    test('it returns "isOperator" if passed in field is "undefined"', () => {
-      const operator = getOperators(undefined);
-
-      expect(operator).toEqual([isOperator]);
-    });
-
-    test('it returns expected operators when field type is "boolean"', () => {
-      const operator = getOperators(getField('ssl'));
-
-      expect(operator).toEqual([isOperator, isNotOperator, existsOperator, doesNotExistOperator]);
-    });
-
-    test('it returns "isOperator" when field type is "nested"', () => {
-      const operator = getOperators({
-        name: 'nestedField',
-        type: 'nested',
-        esTypes: ['text'],
-        count: 0,
-        scripted: false,
-        searchable: true,
-        aggregatable: false,
-        readFromDocValues: false,
-        subType: { nested: { path: 'nestedField' } },
-      });
-
-      expect(operator).toEqual([isOperator]);
-    });
-
-    test('it returns all operator types when field type is not null, boolean, or nested', () => {
-      const operator = getOperators(getField('machine.os.raw'));
-
-      expect(operator).toEqual(EXCEPTION_OPERATORS);
-    });
-  });
 
   describe('#checkEmptyValue', () => {
     test('returns no errors if no field has been selected', () => {
@@ -272,117 +220,4 @@ describe('helpers', () => {
       });
     });
   });
-
-  describe('#typeMatch', () => {
-    test('ip -> ip is true', () => {
-      expect(typeMatch('ip', 'ip')).toEqual(true);
-    });
-
-    test('keyword -> keyword is true', () => {
-      expect(typeMatch('keyword', 'keyword')).toEqual(true);
-    });
-
-    test('text -> text is true', () => {
-      expect(typeMatch('text', 'text')).toEqual(true);
-    });
-
-    test('ip_range -> ip is true', () => {
-      expect(typeMatch('ip_range', 'ip')).toEqual(true);
-    });
-
-    test('date_range -> date is true', () => {
-      expect(typeMatch('date_range', 'date')).toEqual(true);
-    });
-
-    test('double_range -> double is true', () => {
-      expect(typeMatch('double_range', 'double')).toEqual(true);
-    });
-
-    test('float_range -> float is true', () => {
-      expect(typeMatch('float_range', 'float')).toEqual(true);
-    });
-
-    test('integer_range -> integer is true', () => {
-      expect(typeMatch('integer_range', 'integer')).toEqual(true);
-    });
-
-    test('long_range -> long is true', () => {
-      expect(typeMatch('long_range', 'long')).toEqual(true);
-    });
-
-    test('ip -> date is false', () => {
-      expect(typeMatch('ip', 'date')).toEqual(false);
-    });
-
-    test('long -> float is false', () => {
-      expect(typeMatch('long', 'float')).toEqual(false);
-    });
-
-    test('integer -> long is false', () => {
-      expect(typeMatch('integer', 'long')).toEqual(false);
-    });
-  });
-
-  describe('#filterFieldToList', () => {
-    test('it returns empty array if given a undefined for field', () => {
-      const filter = filterFieldToList([], undefined);
-      expect(filter).toEqual([]);
-    });
-
-    test('it returns empty array if filed does not contain esTypes', () => {
-      const field: IFieldType = { name: 'some-name', type: 'some-type' };
-      const filter = filterFieldToList([], field);
-      expect(filter).toEqual([]);
-    });
-
-    test('it returns single filtered list of ip_range -> ip', () => {
-      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
-      const listItem: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-      const filter = filterFieldToList([listItem], field);
-      const expected: ListSchema[] = [listItem];
-      expect(filter).toEqual(expected);
-    });
-
-    test('it returns single filtered list of ip -> ip', () => {
-      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
-      const listItem: ListSchema = { ...getListResponseMock(), type: 'ip' };
-      const filter = filterFieldToList([listItem], field);
-      const expected: ListSchema[] = [listItem];
-      expect(filter).toEqual(expected);
-    });
-
-    test('it returns single filtered list of keyword -> keyword', () => {
-      const field: IFieldType = { name: 'some-name', type: 'keyword', esTypes: ['keyword'] };
-      const listItem: ListSchema = { ...getListResponseMock(), type: 'keyword' };
-      const filter = filterFieldToList([listItem], field);
-      const expected: ListSchema[] = [listItem];
-      expect(filter).toEqual(expected);
-    });
-
-    test('it returns single filtered list of text -> text', () => {
-      const field: IFieldType = { name: 'some-name', type: 'text', esTypes: ['text'] };
-      const listItem: ListSchema = { ...getListResponseMock(), type: 'text' };
-      const filter = filterFieldToList([listItem], field);
-      const expected: ListSchema[] = [listItem];
-      expect(filter).toEqual(expected);
-    });
-
-    test('it returns 2 filtered lists of ip_range -> ip', () => {
-      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
-      const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-      const listItem2: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-      const filter = filterFieldToList([listItem1, listItem2], field);
-      const expected: ListSchema[] = [listItem1, listItem2];
-      expect(filter).toEqual(expected);
-    });
-
-    test('it returns 1 filtered lists of ip_range -> ip if the 2nd is not compatible type', () => {
-      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
-      const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-      const listItem2: ListSchema = { ...getListResponseMock(), type: 'text' };
-      const filter = filterFieldToList([listItem1, listItem2], field);
-      const expected: ListSchema[] = [listItem1];
-      expect(filter).toEqual(expected);
-    });
-  });
 });
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts
index 81f5a66238567d..890f1e67558349 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts
@@ -8,46 +8,17 @@
 import dateMath from '@elastic/datemath';
 import { EuiComboBoxOptionOption } from '@elastic/eui';
 
-import type { Type, ListSchema } from '@kbn/securitysolution-io-ts-list-types';
-import {
-  EXCEPTION_OPERATORS,
-  isOperator,
-  isNotOperator,
-  existsOperator,
-  doesNotExistOperator,
-} from '@kbn/securitysolution-list-utils';
 import { IFieldType } from '../../../../../../../src/plugins/data/common';
 
-import { GetGenericComboBoxPropsReturn, OperatorOption } from './types';
+import { GetGenericComboBoxPropsReturn } from './types';
 import * as i18n from './translations';
 
-/**
- * Returns the appropriate operators given a field type
- *
- * @param field IFieldType selected field
- *
- */
-export const getOperators = (field: IFieldType | undefined): OperatorOption[] => {
-  if (field == null) {
-    return [isOperator];
-  } else if (field.type === 'boolean') {
-    return [isOperator, isNotOperator, existsOperator, doesNotExistOperator];
-  } else if (field.type === 'nested') {
-    return [isOperator];
-  } else {
-    return EXCEPTION_OPERATORS;
-  }
-};
-
 /**
  * Determines if empty value is ok
+ * There is a copy within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
  *
- * @param param the value being checked
- * @param field the selected field
- * @param isRequired whether or not an empty value is allowed
- * @param touched has field been touched by user
- * @returns undefined if valid, string with error message if invalid,
- * null if no checks matched
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  */
 export const checkEmptyValue = (
   param: string | undefined,
@@ -72,7 +43,10 @@ export const checkEmptyValue = (
 
 /**
  * Very basic validation for values
+ * There is a copy within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
  *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  * @param param the value being checked
  * @param field the selected field
  * @param isRequired whether or not an empty value is allowed
@@ -109,7 +83,10 @@ export const paramIsValid = (
 
 /**
  * Determines the options, selected values and option labels for EUI combo box
+ * There is a copy within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/helpers.ts
  *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  * @param options options user can select from
  * @param selectedOptions user selection if any
  * @param getLabel helper function to know which property to use for labels
@@ -140,36 +117,3 @@ export function getGenericComboBoxProps<T>({
     selectedComboOptions: newSelectedComboOptions,
   };
 }
-
-/**
- * Given an array of lists and optionally a field this will return all
- * the lists that match against the field based on the types from the field
- * @param lists The lists to match against the field
- * @param field The field to check against the list to see if they are compatible
- */
-export const filterFieldToList = (lists: ListSchema[], field?: IFieldType): ListSchema[] => {
-  if (field != null) {
-    const { esTypes = [] } = field;
-    return lists.filter(({ type }) => esTypes.some((esType) => typeMatch(type, esType)));
-  } else {
-    return [];
-  }
-};
-
-/**
- * Given an input list type and a string based ES type this will match
- * if they're exact or if they are compatible with a range
- * @param type The type to match against the esType
- * @param esType The ES type to match with
- */
-export const typeMatch = (type: Type, esType: string): boolean => {
-  return (
-    type === esType ||
-    (type === 'ip_range' && esType === 'ip') ||
-    (type === 'date_range' && esType === 'date') ||
-    (type === 'double_range' && esType === 'double') ||
-    (type === 'float_range' && esType === 'float') ||
-    (type === 'integer_range' && esType === 'integer') ||
-    (type === 'long_range' && esType === 'long')
-  );
-};
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/hooks/use_field_value_autocomplete.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/hooks/use_field_value_autocomplete.ts
index 0f369fa01d01ed..0fc4a663b7e11b 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/hooks/use_field_value_autocomplete.ts
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/hooks/use_field_value_autocomplete.ts
@@ -30,9 +30,13 @@ export interface UseFieldValueAutocompleteProps {
   query: string;
   indexPattern: IIndexPattern | undefined;
 }
+
 /**
  * Hook for using the field value autocomplete service
+ * There is a copy within:
+ * x-pack/plugins/lists/public/exceptions/components/autocomplete/hooks.ts
  *
+ * TODO: This should be in its own packaged and not copied, https://github.com/elastic/kibana/issues/105378
  */
 export const useFieldValueAutocomplete = ({
   selectedField,
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.test.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.test.tsx
deleted file mode 100644
index 5e00d2beb571c4..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.test.tsx
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React from 'react';
-import { mount } from 'enzyme';
-import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
-
-import { getField } from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks';
-import { OperatorComponent } from './operator';
-import { isOperator, isNotOperator } from '@kbn/securitysolution-list-utils';
-
-describe('OperatorComponent', () => {
-  test('it renders disabled if "isDisabled" is true', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={true}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"] input`).prop('disabled')
-    ).toBeTruthy();
-  });
-
-  test('it renders loading if "isLoading" is true', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={true}
-        isClearable={false}
-        onChange={jest.fn()}
-      />
-    );
-    wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"] button`).at(0).simulate('click');
-    expect(
-      wrapper
-        .find(`EuiComboBoxOptionsList[data-test-subj="operatorAutocompleteComboBox-optionsList"]`)
-        .prop('isLoading')
-    ).toBeTruthy();
-  });
-
-  test('it allows user to clear values if "isClearable" is true', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={true}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(wrapper.find(`button[data-test-subj="comboBoxClearButton"]`).exists()).toBeTruthy();
-  });
-
-  test('it displays "operatorOptions" if param is passed in with items', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-        operatorOptions={[isNotOperator]}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"]`).at(0).prop('options')
-    ).toEqual([{ label: 'is not' }]);
-  });
-
-  test('it does not display "operatorOptions" if param is passed in with no items', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-        operatorOptions={[]}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"]`).at(0).prop('options')
-    ).toEqual([
-      {
-        label: 'is',
-      },
-      {
-        label: 'is not',
-      },
-      {
-        label: 'is one of',
-      },
-      {
-        label: 'is not one of',
-      },
-      {
-        label: 'exists',
-      },
-      {
-        label: 'does not exist',
-      },
-      {
-        label: 'is in list',
-      },
-      {
-        label: 'is not in list',
-      },
-    ]);
-  });
-
-  test('it correctly displays selected operator', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"] EuiComboBoxPill`).at(0).text()
-    ).toEqual('is');
-  });
-
-  test('it only displays subset of operators if field type is nested', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={{
-          name: 'nestedField',
-          type: 'nested',
-          esTypes: ['text'],
-          count: 0,
-          scripted: false,
-          searchable: true,
-          aggregatable: false,
-          readFromDocValues: false,
-          subType: { nested: { path: 'nestedField' } },
-        }}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"]`).at(0).prop('options')
-    ).toEqual([{ label: 'is' }]);
-  });
-
-  test('it only displays subset of operators if field type is boolean', () => {
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('ssl')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={jest.fn()}
-      />
-    );
-
-    expect(
-      wrapper.find(`[data-test-subj="operatorAutocompleteComboBox"]`).at(0).prop('options')
-    ).toEqual([
-      { label: 'is' },
-      { label: 'is not' },
-      { label: 'exists' },
-      { label: 'does not exist' },
-    ]);
-  });
-
-  test('it invokes "onChange" when option selected', () => {
-    const mockOnChange = jest.fn();
-    const wrapper = mount(
-      <OperatorComponent
-        placeholder="Placeholder text"
-        selectedField={getField('machine.os.raw')}
-        operator={isOperator}
-        isDisabled={false}
-        isLoading={false}
-        isClearable={false}
-        onChange={mockOnChange}
-      />
-    );
-
-    ((wrapper.find(EuiComboBox).props() as unknown) as {
-      onChange: (a: EuiComboBoxOptionOption[]) => void;
-    }).onChange([{ label: 'is not' }]);
-
-    expect(mockOnChange).toHaveBeenCalledWith([
-      { message: 'is not', operator: 'excluded', type: 'match', value: 'is_not' },
-    ]);
-  });
-});
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.tsx
deleted file mode 100644
index d8f49acd04b99b..00000000000000
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/operator.tsx
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React, { useCallback, useMemo } from 'react';
-import { EuiComboBoxOptionOption, EuiComboBox } from '@elastic/eui';
-
-import { IFieldType } from '../../../../../../../src/plugins/data/common';
-import { getOperators, getGenericComboBoxProps } from './helpers';
-import { GetGenericComboBoxPropsReturn, OperatorOption } from './types';
-
-interface OperatorState {
-  placeholder: string;
-  selectedField: IFieldType | undefined;
-  operator: OperatorOption;
-  isLoading: boolean;
-  isDisabled: boolean;
-  isClearable: boolean;
-  operatorInputWidth?: number;
-  operatorOptions?: OperatorOption[];
-  onChange: (arg: OperatorOption[]) => void;
-}
-
-export const OperatorComponent: React.FC<OperatorState> = ({
-  placeholder,
-  selectedField,
-  operator,
-  isLoading = false,
-  isDisabled = false,
-  isClearable = false,
-  operatorOptions,
-  operatorInputWidth = 150,
-  onChange,
-}): JSX.Element => {
-  const getLabel = useCallback(({ message }): string => message, []);
-  const optionsMemo = useMemo(
-    (): OperatorOption[] =>
-      operatorOptions != null && operatorOptions.length > 0
-        ? operatorOptions
-        : getOperators(selectedField),
-    [operatorOptions, selectedField]
-  );
-  const selectedOptionsMemo = useMemo((): OperatorOption[] => (operator ? [operator] : []), [
-    operator,
-  ]);
-  const { comboOptions, labels, selectedComboOptions } = useMemo(
-    (): GetGenericComboBoxPropsReturn =>
-      getGenericComboBoxProps<OperatorOption>({
-        options: optionsMemo,
-        selectedOptions: selectedOptionsMemo,
-        getLabel,
-      }),
-    [optionsMemo, selectedOptionsMemo, getLabel]
-  );
-
-  const handleValuesChange = (newOptions: EuiComboBoxOptionOption[]): void => {
-    const newValues: OperatorOption[] = newOptions.map(
-      ({ label }) => optionsMemo[labels.indexOf(label)]
-    );
-    onChange(newValues);
-  };
-
-  return (
-    <EuiComboBox
-      placeholder={placeholder}
-      options={comboOptions}
-      selectedOptions={selectedComboOptions}
-      onChange={handleValuesChange}
-      isLoading={isLoading}
-      isDisabled={isDisabled}
-      isClearable={isClearable}
-      singleSelection={{ asPlainText: true }}
-      data-test-subj="operatorAutocompleteComboBox"
-      style={{ width: `${operatorInputWidth}px` }}
-    />
-  );
-};
-
-OperatorComponent.displayName = 'Operator';
diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/types.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/types.ts
index 1d8e3e9aee28eb..07f1903fb70e1c 100644
--- a/x-pack/plugins/security_solution/public/common/components/autocomplete/types.ts
+++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/types.ts
@@ -7,20 +7,8 @@
 
 import { EuiComboBoxOptionOption } from '@elastic/eui';
 
-import type {
-  ListOperatorEnum as OperatorEnum,
-  ListOperatorTypeEnum as OperatorTypeEnum,
-} from '@kbn/securitysolution-io-ts-list-types';
-
 export interface GetGenericComboBoxPropsReturn {
   comboOptions: EuiComboBoxOptionOption[];
   labels: string[];
   selectedComboOptions: EuiComboBoxOptionOption[];
 }
-
-export interface OperatorOption {
-  message: string;
-  value: string;
-  operator: OperatorEnum;
-  type: OperatorTypeEnum;
-}

From 194725d351a601b2c3fdecd17b9f2a0e857220f3 Mon Sep 17 00:00:00 2001
From: Oliver Gupte <ogupte@users.noreply.github.com>
Date: Wed, 14 Jul 2021 00:11:38 -0400
Subject: [PATCH 16/27] [APM] Fixes support for APM data streams in Security
 and Timelines UIs (#105334)

* addes support for apm data streams (traces-apm*) to timelines

* [APM] Adds support for apm data streams (traces-apm*) to security solution (#94702)

* fix unit tests

* reverting prepackaged_rules changes to be bumped later
---
 .../security_solution/common/constants.ts     |  1 +
 .../security_solution/cypress/objects/rule.ts |  1 +
 .../drag_drop_context_wrapper.test.tsx.snap   |  1 +
 .../event_details/__mocks__/index.ts          |  2 ++
 .../components/sourcerer/index.test.tsx       |  1 +
 .../common/store/sourcerer/selectors.test.ts  |  3 ++
 .../detection_engine/alerts/mock.ts           |  4 +++
 .../containers/detection_engine/rules/mock.ts |  1 +
 .../detection_engine/rules/use_rule.test.tsx  |  1 +
 .../rules/use_rule_status.test.tsx            |  1 +
 .../rules/use_rule_with_fallback.test.tsx     |  2 ++
 .../components/embeddables/__mocks__/mock.ts  | 27 ++++++++++++++++
 .../embeddables/embedded_map_helpers.test.tsx |  9 +++++-
 .../components/embeddables/map_config.ts      | 31 ++++++++++---------
 .../__snapshots__/index.test.tsx.snap         |  1 +
 .../suricata_row_renderer.test.tsx.snap       |  1 +
 .../__snapshots__/zeek_details.test.tsx.snap  |  1 +
 .../zeek_row_renderer.test.tsx.snap           |  1 +
 .../detections_admin/detections_role.json     |  1 +
 .../roles_users/hunter/detections_role.json   |  1 +
 .../platform_engineer/detections_role.json    |  1 +
 .../rule_author/detections_role.json          |  1 +
 .../soc_manager/detections_role.json          |  1 +
 .../t1_analyst/detections_role.json           |  1 +
 .../t2_analyst/detections_role.json           |  1 +
 .../rules/patches/simplest_updated_name.json  |  1 +
 .../rules/queries/query_with_mappings.json    |  1 +
 .../queries/action_without_meta.json          |  1 +
 .../source_status/elasticsearch_adapter.ts    |  8 +++--
 .../factory/hosts/all/__mocks__/index.ts      |  3 ++
 .../hosts/authentications/__mocks__/index.ts  |  3 ++
 .../factory/hosts/details/__mocks__/index.ts  |  3 ++
 .../hosts/last_first_seen/__mocks__/index.ts  |  4 +++
 .../factory/hosts/overview/__mocks__/index.ts |  3 ++
 .../uncommon_processes/__mocks__/index.ts     |  3 ++
 .../matrix_histogram/__mocks__/index.ts       |  6 ++++
 .../alerts/__mocks__/index.ts                 |  2 ++
 .../anomalies/__mocks__/index.ts              |  2 ++
 .../authentications/__mocks__/index.ts        |  2 ++
 .../matrix_histogram/dns/__mocks__/index.ts   |  2 ++
 .../events/__mocks__/index.ts                 |  9 ++++++
 .../network/details/__mocks__/index.ts        |  3 ++
 .../factory/network/dns/__mocks__/index.ts    |  3 ++
 .../factory/network/http/__mocks__/index.ts   |  3 ++
 .../network/overview/__mocks__/index.ts       |  3 ++
 .../factory/network/tls/__mocks__/index.ts    |  3 ++
 .../network/top_countries/__mocks__/index.ts  |  3 ++
 .../network/top_n_flow/__mocks__/index.ts     |  3 ++
 .../factory/network/users/__mocks__/index.ts  |  3 ++
 .../__snapshots__/index.test.tsx.snap         |  1 +
 .../timelines/public/mock/browser_fields.ts   |  1 +
 .../timelines/public/mock/global_state.ts     |  1 +
 .../index_fields/index.test.ts                | 18 ++++++++---
 .../search_strategy/index_fields/index.ts     |  6 +++-
 .../factory/events/all/helpers.test.ts        |  9 ++++--
 55 files changed, 183 insertions(+), 26 deletions(-)

diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
index abfcb4014a79fd..27d4a5c9fd3994 100644
--- a/x-pack/plugins/security_solution/common/constants.ts
+++ b/x-pack/plugins/security_solution/common/constants.ts
@@ -110,6 +110,7 @@ export const APP_EVENT_FILTERS_PATH = `${APP_PATH}${EVENT_FILTERS_PATH}`;
 /** The comma-delimited list of Elasticsearch indices from which the SIEM app collects events */
 export const DEFAULT_INDEX_PATTERN = [
   'apm-*-transaction*',
+  'traces-apm*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',
diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts
index a10fa5b0eda78c..7589c8fab3dae5 100644
--- a/x-pack/plugins/security_solution/cypress/objects/rule.ts
+++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts
@@ -98,6 +98,7 @@ export interface MachineLearningRule {
 
 export const getIndexPatterns = (): string[] => [
   'apm-*-transaction*',
+  'traces-apm*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap
index 2dc3f7fd336a2d..6ef797580be9b3 100644
--- a/x-pack/plugins/security_solution/public/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap
+++ b/x-pack/plugins/security_solution/public/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap
@@ -366,6 +366,7 @@ exports[`DragDropContextWrapper rendering it renders against the snapshot 1`] =
             "format": "",
             "indexes": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts b/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts
index 3edd6e6fda14b3..620c3991b0ad98 100644
--- a/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts
@@ -406,6 +406,7 @@ export const mockAlertDetailsData = [
     field: 'signal.rule.index',
     values: [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',
@@ -415,6 +416,7 @@ export const mockAlertDetailsData = [
     ],
     originalValue: [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx
index e8f382a5050d82..87a7ce805940f7 100644
--- a/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx
+++ b/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx
@@ -39,6 +39,7 @@ const mockOptions = [
   { label: 'filebeat-*', value: 'filebeat-*' },
   { label: 'logs-*', value: 'logs-*' },
   { label: 'packetbeat-*', value: 'packetbeat-*' },
+  { label: 'traces-apm*', value: 'traces-apm*' },
   { label: 'winlogbeat-*', value: 'winlogbeat-*' },
 ];
 
diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts
index 730857b6494d9b..dd608138ef9f03 100644
--- a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts
+++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts
@@ -23,6 +23,7 @@ describe('Sourcerer selectors', () => {
         'filebeat-*',
         'logs-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
         '-*elastic-cloud-logs-*',
       ]);
@@ -42,6 +43,7 @@ describe('Sourcerer selectors', () => {
         'endgame-*',
         'filebeat-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
       ]);
     });
@@ -64,6 +66,7 @@ describe('Sourcerer selectors', () => {
         'filebeat-*',
         'logs-endpoint.event-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
       ]);
     });
diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts
index e4bddfba8278bb..7aba8fa4ac10f1 100644
--- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts
+++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts
@@ -175,6 +175,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -414,6 +415,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -619,6 +621,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -822,6 +825,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts
index 1104cb86064b07..533ab6138cb09f 100644
--- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts
+++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts
@@ -20,6 +20,7 @@ export const savedRuleMock: Rule = {
   id: '12345678987654321',
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx
index ca6cd5b11f7057..096463872fc011 100644
--- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx
+++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx
@@ -54,6 +54,7 @@ describe('useRule', () => {
           immutable: false,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx
index 3394f1fc553ae5..4d01e2ff00ec18 100644
--- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx
+++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx
@@ -43,6 +43,7 @@ const testRule: Rule = {
   immutable: false,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx
index abd5a2781c8a77..1f08a356602152 100644
--- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx
+++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx
@@ -62,6 +62,7 @@ describe('useRuleWithFallback', () => {
             "immutable": false,
             "index": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
@@ -125,6 +126,7 @@ describe('useRuleWithFallback', () => {
             "immutable": false,
             "index": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts
index ba9b6518c6accc..834447b21929fe 100644
--- a/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts
+++ b/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts
@@ -14,6 +14,7 @@ export const mockIndexPatternIds: IndexPatternMapping[] = [
 
 export const mockAPMIndexPatternIds: IndexPatternMapping[] = [
   { title: 'apm-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
+  { title: 'traces-apm*,logs-apm*,metrics-apm*,apm-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
 ];
 
 export const mockSourceLayer = {
@@ -183,6 +184,11 @@ export const mockClientLayer = {
   joins: [],
 };
 
+const mockApmDataStreamClientLayer = {
+  ...mockClientLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Client Point',
+};
+
 export const mockServerLayer = {
   sourceDescriptor: {
     id: 'uuid.v4()',
@@ -238,6 +244,11 @@ export const mockServerLayer = {
   query: { query: '', language: 'kuery' },
 };
 
+const mockApmDataStreamServerLayer = {
+  ...mockServerLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Server Point',
+};
+
 export const mockLineLayer = {
   sourceDescriptor: {
     type: 'ES_PEW_PEW',
@@ -365,6 +376,10 @@ export const mockClientServerLineLayer = {
   type: 'VECTOR',
   query: { query: '', language: 'kuery' },
 };
+const mockApmDataStreamClientServerLineLayer = {
+  ...mockClientServerLineLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Line',
+};
 
 export const mockLayerList = [
   {
@@ -421,6 +436,9 @@ export const mockLayerListMixed = [
   mockClientServerLineLayer,
   mockServerLayer,
   mockClientLayer,
+  mockApmDataStreamClientServerLineLayer,
+  mockApmDataStreamServerLayer,
+  mockApmDataStreamClientLayer,
 ];
 
 export const mockAPMIndexPattern: IndexPatternSavedObject = {
@@ -468,6 +486,15 @@ export const mockAPMTransactionIndexPattern: IndexPatternSavedObject = {
   },
 };
 
+export const mockAPMTracesDataStreamIndexPattern: IndexPatternSavedObject = {
+  id: 'traces-apm*',
+  type: 'index-pattern',
+  _version: 'abc',
+  attributes: {
+    title: 'traces-apm*',
+  },
+};
+
 export const mockGlobIndexPattern: IndexPatternSavedObject = {
   id: '*',
   type: 'index-pattern',
diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx b/x-pack/plugins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx
index 6136f5da51d51a..613a6ce4c00daa 100644
--- a/x-pack/plugins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx
+++ b/x-pack/plugins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx
@@ -12,6 +12,7 @@ import {
   mockAPMIndexPattern,
   mockAPMRegexIndexPattern,
   mockAPMTransactionIndexPattern,
+  mockAPMTracesDataStreamIndexPattern,
   mockAuditbeatIndexPattern,
   mockCCSGlobIndexPattern,
   mockCommaFilebeatAuditbeatCCSGlobIndexPattern,
@@ -69,6 +70,7 @@ describe('embedded_map_helpers', () => {
   describe('findMatchingIndexPatterns', () => {
     const siemDefaultIndices = [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',
@@ -102,11 +104,16 @@ describe('embedded_map_helpers', () => {
 
     test('finds exact glob-matched index patterns ', () => {
       const matchingIndexPatterns = findMatchingIndexPatterns({
-        kibanaIndexPatterns: [mockAPMTransactionIndexPattern, mockFilebeatIndexPattern],
+        kibanaIndexPatterns: [
+          mockAPMTransactionIndexPattern,
+          mockAPMTracesDataStreamIndexPattern,
+          mockFilebeatIndexPattern,
+        ],
         siemDefaultIndices,
       });
       expect(matchingIndexPatterns).toEqual([
         mockAPMTransactionIndexPattern,
+        mockAPMTracesDataStreamIndexPattern,
         mockFilebeatIndexPattern,
       ]);
     });
diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts b/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts
index f4af4dd3b25f2b..ecbb80123e07ec 100644
--- a/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts
+++ b/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts
@@ -61,6 +61,21 @@ export const SUM_OF_DESTINATION_BYTES = 'sum_of_destination.bytes';
 export const SUM_OF_CLIENT_BYTES = 'sum_of_client.bytes';
 export const SUM_OF_SERVER_BYTES = 'sum_of_server.bytes';
 
+const APM_LAYER_FIELD_MAPPING = {
+  source: {
+    metricField: 'client.bytes',
+    geoField: 'client.geo.location',
+    tooltipProperties: Object.keys(clientFieldMappings),
+    label: i18n.CLIENT_LAYER,
+  },
+  destination: {
+    metricField: 'server.bytes',
+    geoField: 'server.geo.location',
+    tooltipProperties: Object.keys(serverFieldMappings),
+    label: i18n.SERVER_LAYER,
+  },
+};
+
 // Mapping to fields for creating specific layers for a given index pattern
 // e.g. The apm-* index pattern needs layers for client/server instead of source/destination
 export const lmc: LayerMappingCollection = {
@@ -78,20 +93,8 @@ export const lmc: LayerMappingCollection = {
       label: i18n.DESTINATION_LAYER,
     },
   },
-  'apm-*': {
-    source: {
-      metricField: 'client.bytes',
-      geoField: 'client.geo.location',
-      tooltipProperties: Object.keys(clientFieldMappings),
-      label: i18n.CLIENT_LAYER,
-    },
-    destination: {
-      metricField: 'server.bytes',
-      geoField: 'server.geo.location',
-      tooltipProperties: Object.keys(serverFieldMappings),
-      label: i18n.SERVER_LAYER,
-    },
-  },
+  'apm-*': APM_LAYER_FIELD_MAPPING,
+  'traces-apm*,logs-apm*,metrics-apm*,apm-*': APM_LAYER_FIELD_MAPPING,
 };
 
 /**
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap
index d484a76940d5f4..b008f95285f236 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap
@@ -367,6 +367,7 @@ exports[`ColumnHeaders rendering renders correctly against snapshot 1`] = `
             "format": "",
             "indexes": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap
index b5bc87ff636b3c..2934d35dc184da 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap
@@ -368,6 +368,7 @@ exports[`suricata_row_renderer renders correctly against snapshot 1`] = `
                 "format": "",
                 "indexes": Array [
                   "apm-*-transaction*",
+                  "traces-apm*",
                   "auditbeat-*",
                   "endgame-*",
                   "filebeat-*",
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap
index bd50554ea36120..64ca766e4dee6a 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap
@@ -366,6 +366,7 @@ exports[`ZeekDetails rendering it renders the default ZeekDetails 1`] = `
             "format": "",
             "indexes": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap
index a136ac80ec7e3d..6c59df606cd36b 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap
@@ -368,6 +368,7 @@ exports[`zeek_row_renderer renders correctly against snapshot 1`] = `
                 "format": "",
                 "indexes": Array [
                   "apm-*-transaction*",
+                  "traces-apm*",
                   "auditbeat-*",
                   "endgame-*",
                   "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json
index 6c9b4e2cba49c6..82ef8cc6687b44 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json
@@ -8,6 +8,7 @@
           ".lists*",
           ".items*",
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json
index 119fe5421c86c5..ba9adfda82beaa 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json
index 17dbd90d179253..73a9559389b4e5 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json
@@ -9,6 +9,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json
index 0db8359c577640..bb606616d1bd53 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json
index 6962701ae5be35..92a62034afcefc 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json
index 07827069dbc739..be082e380211a0 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json
@@ -6,6 +6,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json
index f554c916c6684d..f9e069f174a91c 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json
@@ -8,6 +8,7 @@
           ".lists*",
           ".items*",
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json
index bec88bcb0e30e7..3e5a5c274f1baa 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json
@@ -8,6 +8,7 @@
   "from": "now-360s",
   "index": [
     "apm-*-transaction*",
+    "traces-apm*",
     "auditbeat-*",
     "endgame-*",
     "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json
index f0d7cb4ec914b4..2508c9a6a3e374 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json
@@ -3,6 +3,7 @@
   "enabled": false,
   "index": [
     "apm-*-transaction*",
+    "traces-apm*",
     "auditbeat-*",
     "endgame-*",
     "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/queries/action_without_meta.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/queries/action_without_meta.json
index 6569a641de3a2c..f0ddced46d52e0 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/queries/action_without_meta.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/queries/action_without_meta.json
@@ -2,6 +2,7 @@
   "type": "query",
   "index": [
     "apm-*-transaction*",
+    "traces-apm*",
     "auditbeat-*",
     "endgame-*",
     "filebeat-*",
diff --git a/x-pack/plugins/security_solution/server/lib/source_status/elasticsearch_adapter.ts b/x-pack/plugins/security_solution/server/lib/source_status/elasticsearch_adapter.ts
index b2058a91c0413b..3da0c1675e81eb 100644
--- a/x-pack/plugins/security_solution/server/lib/source_status/elasticsearch_adapter.ts
+++ b/x-pack/plugins/security_solution/server/lib/source_status/elasticsearch_adapter.ts
@@ -12,6 +12,7 @@ import { ApmServiceNameAgg } from './types';
 import { ENDPOINT_METADATA_INDEX } from '../../../common/constants';
 
 const APM_INDEX_NAME = 'apm-*-transaction*';
+const APM_DATA_STREAM = 'traces-apm*';
 
 export class ElasticsearchSourceStatusAdapter implements SourceStatusAdapter {
   constructor(private readonly framework: FrameworkAdapter) {}
@@ -23,7 +24,9 @@ export class ElasticsearchSourceStatusAdapter implements SourceStatusAdapter {
       // Add endpoint metadata index to indices to check
       indexNames.push(ENDPOINT_METADATA_INDEX);
       // Remove APM index if exists, and only query if length > 0 in case it's the only index provided
-      const nonApmIndexNames = indexNames.filter((name) => name !== APM_INDEX_NAME);
+      const nonApmIndexNames = indexNames.filter(
+        (name) => name !== APM_INDEX_NAME && name !== APM_DATA_STREAM
+      );
       const indexCheckResponse = await (nonApmIndexNames.length > 0
         ? this.framework.callWithRequest(request, 'search', {
             index: nonApmIndexNames,
@@ -39,7 +42,8 @@ export class ElasticsearchSourceStatusAdapter implements SourceStatusAdapter {
 
       // Note: Additional check necessary for APM-specific index. For details see: https://github.com/elastic/kibana/issues/56363
       // Only verify if APM data exists if indexNames includes `apm-*-transaction*` (default included apm index)
-      const includesApmIndex = indexNames.includes(APM_INDEX_NAME);
+      const includesApmIndex =
+        indexNames.includes(APM_INDEX_NAME) || indexNames.includes(APM_DATA_STREAM);
       const hasApmDataResponse = await (includesApmIndex
         ? this.framework.callWithRequest<{}, ApmServiceNameAgg>(
             request,
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts
index b6a5435a0e0461..0369f182a4c753 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts
@@ -18,6 +18,7 @@ import {
 export const mockOptions: HostsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -613,6 +614,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -822,6 +824,7 @@ export const expectedDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts
index f29bb58da2f790..1dd3dc8ee4cff5 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts
@@ -17,6 +17,7 @@ import {
 export const mockOptions: HostAuthenticationsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -2151,6 +2152,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -2372,6 +2374,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts
index 9dfff5e11715d0..cc97a5f0cacefa 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts
@@ -17,6 +17,7 @@ import {
 export const mockOptions: HostDetailsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -1303,6 +1304,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -1416,6 +1418,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts
index b492bf57f94a66..443e7e96a3c7f8 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts
@@ -14,6 +14,7 @@ import {
 export const mockOptions: HostFirstLastSeenRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -126,6 +127,7 @@ export const formattedSearchStrategyFirstResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -191,6 +193,7 @@ export const formattedSearchStrategyLastResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -225,6 +228,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts
index 987754420430d6..2b4e4b8291401a 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts
@@ -15,6 +15,7 @@ import {
 export const mockOptions: HostOverviewRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -119,6 +120,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -331,6 +333,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts
index 258e72a5e9b8eb..0ad976a0f498c2 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts
@@ -10,6 +10,7 @@ import { HostsQueries, SortField } from '../../../../../../../common/search_stra
 export const mockOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -4302,6 +4303,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -4436,6 +4438,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts
index c33ca75aa26e12..7f36e3551e5bee 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts
@@ -33,6 +33,7 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo
         {
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -166,6 +167,7 @@ export const expectedDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -199,6 +201,7 @@ export const formattedAnomaliesSearchStrategyResponse: MatrixHistogramStrategyRe
         {
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -381,6 +384,7 @@ export const formattedAuthenticationsSearchStrategyResponse: MatrixHistogramStra
         {
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -947,6 +951,7 @@ export const formattedEventsSearchStrategyResponse: MatrixHistogramStrategyRespo
         {
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -1925,6 +1930,7 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts
index 86006c31554477..82531f35b09abc 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts
@@ -10,6 +10,7 @@ import { MatrixHistogramType } from '../../../../../../../common/search_strategy
 export const mockOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -27,6 +28,7 @@ export const mockOptions = {
 export const expectedDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts
index 81da78a132084a..ab76d54dee11ff 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts
@@ -10,6 +10,7 @@ import { MatrixHistogramType } from '../../../../../../../common/search_strategy
 export const mockOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -27,6 +28,7 @@ export const mockOptions = {
 export const expectedDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts
index 5cf667a0085fa7..1fd7b85242df64 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts
@@ -10,6 +10,7 @@ import { MatrixHistogramType } from '../../../../../../../common/search_strategy
 export const mockOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -26,6 +27,7 @@ export const mockOptions = {
 export const expectedDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts
index 9b8dfb139d9f40..4d97fba3cb80cf 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts
@@ -10,6 +10,7 @@ import { MatrixHistogramType } from '../../../../../../../common/search_strategy
 export const mockOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -28,6 +29,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts
index c361db38a6caac..5dab2bcd5cf9de 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts
@@ -14,6 +14,7 @@ import {
 export const mockOptions: MatrixHistogramRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -31,6 +32,7 @@ export const mockOptions: MatrixHistogramRequestOptions = {
 export const expectedDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -85,6 +87,7 @@ export const expectedDsl = {
 export const expectedThresholdDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -141,6 +144,7 @@ export const expectedThresholdDsl = {
 export const expectedThresholdMissingFieldDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -243,6 +247,7 @@ export const expectedThresholdWithCardinalityDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -256,6 +261,7 @@ export const expectedThresholdWithCardinalityDsl = {
 export const expectedThresholdWithGroupFieldsAndCardinalityDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -362,6 +368,7 @@ export const expectedThresholdGroupWithCardinalityDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -375,6 +382,7 @@ export const expectedThresholdGroupWithCardinalityDsl = {
 export const expectedIpIncludingMissingDataDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -437,6 +445,7 @@ export const expectedIpIncludingMissingDataDsl = {
 export const expectedIpNotIncludingMissingDataDsl = {
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts
index fb11069f9c8346..7f71906bcaa97f 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts
@@ -15,6 +15,7 @@ import {
 export const mockOptions: NetworkDetailsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -306,6 +307,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -447,6 +449,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts
index 3252a7c249d722..cc01450e5bec56 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts
@@ -17,6 +17,7 @@ import {
 export const mockOptions: NetworkDnsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -133,6 +134,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -204,6 +206,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts
index aaf29f07537b54..b34027338e2ba9 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts
@@ -18,6 +18,7 @@ import {
 export const mockOptions: NetworkHttpRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -621,6 +622,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -678,6 +680,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts
index fcb30be7a403d6..74b201e9a2294b 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts
@@ -15,6 +15,7 @@ import {
 export const mockOptions: NetworkOverviewRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -103,6 +104,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -208,6 +210,7 @@ export const expectedDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts
index 16750acc5adeed..8616a2ef14856f 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts
@@ -18,6 +18,7 @@ import {
 export const mockOptions: NetworkTlsRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -61,6 +62,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -115,6 +117,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts
index 9f95dbe9c1c4f2..ba5db90df82452 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts
@@ -18,6 +18,7 @@ import {
 export const mockOptions: NetworkTopCountriesRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -60,6 +61,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -119,6 +121,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts
index c815ed22f2b548..e881a9ef93949c 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts
@@ -19,6 +19,7 @@ import {
 export const mockOptions: NetworkTopNFlowRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -812,6 +813,7 @@ export const formattedSearchStrategyResponse: NetworkTopNFlowStrategyResponse =
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -879,6 +881,7 @@ export const expectedDsl = {
   allowNoIndices: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts
index 3837afabe57993..686730dbe79275 100644
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts
@@ -18,6 +18,7 @@ import {
 export const mockOptions: NetworkUsersRequestOptions = {
   defaultIndex: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
@@ -121,6 +122,7 @@ export const formattedSearchStrategyResponse = {
           allowNoIndices: true,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',
@@ -210,6 +212,7 @@ export const expectedDsl = {
   ignoreUnavailable: true,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',
diff --git a/x-pack/plugins/timelines/public/components/t_grid/body/column_headers/__snapshots__/index.test.tsx.snap b/x-pack/plugins/timelines/public/components/t_grid/body/column_headers/__snapshots__/index.test.tsx.snap
index 9ee08bcd966f35..e6e56818bcc846 100644
--- a/x-pack/plugins/timelines/public/components/t_grid/body/column_headers/__snapshots__/index.test.tsx.snap
+++ b/x-pack/plugins/timelines/public/components/t_grid/body/column_headers/__snapshots__/index.test.tsx.snap
@@ -367,6 +367,7 @@ exports[`ColumnHeaders rendering renders correctly against snapshot 1`] = `
             "format": "",
             "indexes": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
diff --git a/x-pack/plugins/timelines/public/mock/browser_fields.ts b/x-pack/plugins/timelines/public/mock/browser_fields.ts
index 1581175e329043..6ab06e1be018a6 100644
--- a/x-pack/plugins/timelines/public/mock/browser_fields.ts
+++ b/x-pack/plugins/timelines/public/mock/browser_fields.ts
@@ -10,6 +10,7 @@ import type { BrowserFields } from '../../common/search_strategy/index_fields';
 
 const DEFAULT_INDEX_PATTERN = [
   'apm-*-transaction*',
+  'traces-apm*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',
diff --git a/x-pack/plugins/timelines/public/mock/global_state.ts b/x-pack/plugins/timelines/public/mock/global_state.ts
index bb7bee3d1552ad..f7d3297738373d 100644
--- a/x-pack/plugins/timelines/public/mock/global_state.ts
+++ b/x-pack/plugins/timelines/public/mock/global_state.ts
@@ -24,6 +24,7 @@ export const mockGlobalState: TimelineState = {
       id: 'test',
       indexNames: [
         'apm-*-transaction*',
+        'traces-apm*',
         'auditbeat-*',
         'endgame-*',
         'filebeat-*',
diff --git a/x-pack/plugins/timelines/server/search_strategy/index_fields/index.test.ts b/x-pack/plugins/timelines/server/search_strategy/index_fields/index.test.ts
index f6d78f2f1259fd..cb2b097d787018 100644
--- a/x-pack/plugins/timelines/server/search_strategy/index_fields/index.test.ts
+++ b/x-pack/plugins/timelines/server/search_strategy/index_fields/index.test.ts
@@ -845,7 +845,7 @@ describe('Fields Provider', () => {
     });
 
     it('should search apm index fields', async () => {
-      const indices = ['apm-*-transaction*'];
+      const indices = ['apm-*-transaction*', 'traces-apm*'];
       const request = {
         indices,
         onlyCheckIfIndicesExist: false,
@@ -861,13 +861,13 @@ describe('Fields Provider', () => {
     });
 
     it('should check apm index exists with data', async () => {
-      const indices = ['apm-*-transaction*'];
+      const indices = ['apm-*-transaction*', 'traces-apm*'];
       const request = {
         indices,
         onlyCheckIfIndicesExist: true,
       };
 
-      esClientSearchMock.mockResolvedValueOnce({
+      esClientSearchMock.mockResolvedValue({
         body: { hits: { total: { value: 1 } } },
       });
       const response = await requestIndexFieldSearch(request, deps, beatFields);
@@ -876,6 +876,10 @@ describe('Fields Provider', () => {
         index: indices[0],
         body: { query: { match_all: {} }, size: 0 },
       });
+      expect(esClientSearchMock).toHaveBeenCalledWith({
+        index: indices[1],
+        body: { query: { match_all: {} }, size: 0 },
+      });
       expect(getFieldsForWildcardMock).not.toHaveBeenCalled();
 
       expect(response.indexFields).toHaveLength(0);
@@ -883,13 +887,13 @@ describe('Fields Provider', () => {
     });
 
     it('should check apm index exists with no data', async () => {
-      const indices = ['apm-*-transaction*'];
+      const indices = ['apm-*-transaction*', 'traces-apm*'];
       const request = {
         indices,
         onlyCheckIfIndicesExist: true,
       };
 
-      esClientSearchMock.mockResolvedValueOnce({
+      esClientSearchMock.mockResolvedValue({
         body: { hits: { total: { value: 0 } } },
       });
 
@@ -899,6 +903,10 @@ describe('Fields Provider', () => {
         index: indices[0],
         body: { query: { match_all: {} }, size: 0 },
       });
+      expect(esClientSearchMock).toHaveBeenCalledWith({
+        index: indices[1],
+        body: { query: { match_all: {} }, size: 0 },
+      });
       expect(getFieldsForWildcardMock).not.toHaveBeenCalled();
 
       expect(response.indexFields).toHaveLength(0);
diff --git a/x-pack/plugins/timelines/server/search_strategy/index_fields/index.ts b/x-pack/plugins/timelines/server/search_strategy/index_fields/index.ts
index d100e8db21493f..b6cf4af1561c3f 100644
--- a/x-pack/plugins/timelines/server/search_strategy/index_fields/index.ts
+++ b/x-pack/plugins/timelines/server/search_strategy/index_fields/index.ts
@@ -25,6 +25,7 @@ import {
 } from '../../../common/search_strategy/index_fields';
 
 const apmIndexPattern = 'apm-*-transaction*';
+const apmDataStreamsPattern = 'traces-apm*';
 
 export const indexFieldsProvider = (): ISearchStrategy<
   IndexFieldsStrategyRequest,
@@ -51,7 +52,10 @@ export const requestIndexFieldSearch = async (
   const responsesIndexFields = await Promise.all(
     dedupeIndices
       .map(async (index) => {
-        if (request.onlyCheckIfIndicesExist && index.includes(apmIndexPattern)) {
+        if (
+          request.onlyCheckIfIndicesExist &&
+          (index.includes(apmIndexPattern) || index.includes(apmDataStreamsPattern))
+        ) {
           // for apm index pattern check also if there's data https://github.com/elastic/kibana/issues/90661
           const searchResponse = await esClient.asCurrentUser.search({
             index,
diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.test.ts
index 9197917ad764f8..c9be6582015f1c 100644
--- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.test.ts
+++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/helpers.test.ts
@@ -141,7 +141,7 @@ describe('#formatTimelineData', () => {
           parent: {
             depth: 0,
             index:
-              'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
+              'apm-*-transaction*,traces-apm*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
             id: '0268af90-d8da-576a-9747-2a191519416a',
             type: 'event',
           },
@@ -180,6 +180,7 @@ describe('#formatTimelineData', () => {
             query: '_id :*',
             index: [
               'apm-*-transaction*',
+              'traces-apm*',
               'auditbeat-*',
               'endgame-*',
               'filebeat-*',
@@ -246,7 +247,7 @@ describe('#formatTimelineData', () => {
             {
               depth: 0,
               index:
-                'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
+                'apm-*-transaction*,traces-apm*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
               id: '0268af90-d8da-576a-9747-2a191519416a',
               type: 'event',
             },
@@ -255,7 +256,7 @@ describe('#formatTimelineData', () => {
             {
               depth: 0,
               index:
-                'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
+                'apm-*-transaction*,traces-apm*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,winlogbeat-*',
               id: '0268af90-d8da-576a-9747-2a191519416a',
               type: 'event',
             },
@@ -279,6 +280,7 @@ describe('#formatTimelineData', () => {
         'signal.rule.version': ['1'],
         'signal.rule.index': [
           'apm-*-transaction*',
+          'traces-apm*',
           'auditbeat-*',
           'endgame-*',
           'filebeat-*',
@@ -332,6 +334,7 @@ describe('#formatTimelineData', () => {
               id: ['696c24e0-526d-11eb-836c-e1620268b945'],
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',

From e60f14eb9422b902bed39ef1ed7fb11859983648 Mon Sep 17 00:00:00 2001
From: Oliver Gupte <ogupte@users.noreply.github.com>
Date: Wed, 14 Jul 2021 00:33:27 -0400
Subject: [PATCH 17/27] [APM] Fixes support for APM index pattern with data
 streams (#105360)

* [APM] Fixes support for APM index pattern with data streams (#94702)

* fix unit test

* - Revert "fix unit test"
- tests each component of index pattern title for index matches

This reverts commit a3a27df4b6b57bbe143254736a4dbe7050cb9fc6.
---
 .../create_layer_descriptor.test.ts           |   2 +-
 .../observability/create_layer_descriptor.ts  |   2 +-
 .../security/create_layer_descriptors.test.ts | 339 ++++++++++++++++++
 .../security/create_layer_descriptors.ts      |   6 +-
 4 files changed, 346 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts
index 1ac2690d6bada2..74ab35e6cb360e 100644
--- a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts
+++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts
@@ -54,7 +54,7 @@ describe('createLayerDescriptor', () => {
             applyGlobalTime: true,
             id: '12345',
             indexPatternId: 'apm_static_index_pattern_id',
-            indexPatternTitle: 'apm-*',
+            indexPatternTitle: 'traces-apm*,logs-apm*,metrics-apm*,apm-*',
             metrics: [
               {
                 field: 'transaction.duration.us',
diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts
index adf6f1d7f270d2..0b57afb38d585e 100644
--- a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts
+++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts
@@ -39,7 +39,7 @@ import { getDefaultDynamicProperties } from '../../../styles/vector/vector_style
 
 // redefining APM constant to avoid making maps app depend on APM plugin
 export const APM_INDEX_PATTERN_ID = 'apm_static_index_pattern_id';
-export const APM_INDEX_PATTERN_TITLE = 'apm-*';
+export const APM_INDEX_PATTERN_TITLE = 'traces-apm*,logs-apm*,metrics-apm*,apm-*';
 
 const defaultDynamicProperties = getDefaultDynamicProperties();
 
diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.test.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.test.ts
index 9c6e72fc11d3ad..a3a3e8b20f678b 100644
--- a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.test.ts
+++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.test.ts
@@ -706,4 +706,343 @@ describe('createLayerDescriptor', () => {
       },
     ]);
   });
+
+  test('apm data stream', () => {
+    expect(createSecurityLayerDescriptors('id', 'traces-apm-opbean-node')).toEqual([
+      {
+        __dataRequests: [],
+        alpha: 0.75,
+        id: '12345',
+        includeInFitToBounds: true,
+        joins: [],
+        label: 'traces-apm-opbean-node | Source Point',
+        maxZoom: 24,
+        minZoom: 0,
+        sourceDescriptor: {
+          applyGlobalQuery: true,
+          applyGlobalTime: true,
+          filterByMapBounds: true,
+          geoField: 'client.geo.location',
+          id: '12345',
+          indexPatternId: 'id',
+          scalingType: 'TOP_HITS',
+          sortField: '',
+          sortOrder: 'desc',
+          tooltipProperties: [
+            'host.name',
+            'client.ip',
+            'client.domain',
+            'client.geo.country_iso_code',
+            'client.as.organization.name',
+          ],
+          topHitsSize: 1,
+          topHitsSplitField: 'client.ip',
+          type: 'ES_SEARCH',
+        },
+        style: {
+          isTimeAware: true,
+          properties: {
+            fillColor: {
+              options: {
+                color: '#6092C0',
+              },
+              type: 'STATIC',
+            },
+            icon: {
+              options: {
+                value: 'home',
+              },
+              type: 'STATIC',
+            },
+            iconOrientation: {
+              options: {
+                orientation: 0,
+              },
+              type: 'STATIC',
+            },
+            iconSize: {
+              options: {
+                size: 8,
+              },
+              type: 'STATIC',
+            },
+            labelBorderColor: {
+              options: {
+                color: '#FFFFFF',
+              },
+              type: 'STATIC',
+            },
+            labelBorderSize: {
+              options: {
+                size: 'SMALL',
+              },
+            },
+            labelColor: {
+              options: {
+                color: '#000000',
+              },
+              type: 'STATIC',
+            },
+            labelSize: {
+              options: {
+                size: 14,
+              },
+              type: 'STATIC',
+            },
+            labelText: {
+              options: {
+                value: '',
+              },
+              type: 'STATIC',
+            },
+            lineColor: {
+              options: {
+                color: '#FFFFFF',
+              },
+              type: 'STATIC',
+            },
+            lineWidth: {
+              options: {
+                size: 2,
+              },
+              type: 'STATIC',
+            },
+            symbolizeAs: {
+              options: {
+                value: 'icon',
+              },
+            },
+          },
+          type: 'VECTOR',
+        },
+        type: 'VECTOR',
+        visible: true,
+      },
+      {
+        __dataRequests: [],
+        alpha: 0.75,
+        id: '12345',
+        includeInFitToBounds: true,
+        joins: [],
+        label: 'traces-apm-opbean-node | Destination point',
+        maxZoom: 24,
+        minZoom: 0,
+        sourceDescriptor: {
+          applyGlobalQuery: true,
+          applyGlobalTime: true,
+          filterByMapBounds: true,
+          geoField: 'server.geo.location',
+          id: '12345',
+          indexPatternId: 'id',
+          scalingType: 'TOP_HITS',
+          sortField: '',
+          sortOrder: 'desc',
+          tooltipProperties: [
+            'host.name',
+            'server.ip',
+            'server.domain',
+            'server.geo.country_iso_code',
+            'server.as.organization.name',
+          ],
+          topHitsSize: 1,
+          topHitsSplitField: 'server.ip',
+          type: 'ES_SEARCH',
+        },
+        style: {
+          isTimeAware: true,
+          properties: {
+            fillColor: {
+              options: {
+                color: '#D36086',
+              },
+              type: 'STATIC',
+            },
+            icon: {
+              options: {
+                value: 'marker',
+              },
+              type: 'STATIC',
+            },
+            iconOrientation: {
+              options: {
+                orientation: 0,
+              },
+              type: 'STATIC',
+            },
+            iconSize: {
+              options: {
+                size: 8,
+              },
+              type: 'STATIC',
+            },
+            labelBorderColor: {
+              options: {
+                color: '#FFFFFF',
+              },
+              type: 'STATIC',
+            },
+            labelBorderSize: {
+              options: {
+                size: 'SMALL',
+              },
+            },
+            labelColor: {
+              options: {
+                color: '#000000',
+              },
+              type: 'STATIC',
+            },
+            labelSize: {
+              options: {
+                size: 14,
+              },
+              type: 'STATIC',
+            },
+            labelText: {
+              options: {
+                value: '',
+              },
+              type: 'STATIC',
+            },
+            lineColor: {
+              options: {
+                color: '#FFFFFF',
+              },
+              type: 'STATIC',
+            },
+            lineWidth: {
+              options: {
+                size: 2,
+              },
+              type: 'STATIC',
+            },
+            symbolizeAs: {
+              options: {
+                value: 'icon',
+              },
+            },
+          },
+          type: 'VECTOR',
+        },
+        type: 'VECTOR',
+        visible: true,
+      },
+      {
+        __dataRequests: [],
+        alpha: 0.75,
+        id: '12345',
+        includeInFitToBounds: true,
+        joins: [],
+        label: 'traces-apm-opbean-node | Line',
+        maxZoom: 24,
+        minZoom: 0,
+        sourceDescriptor: {
+          applyGlobalQuery: true,
+          applyGlobalTime: true,
+          destGeoField: 'server.geo.location',
+          id: '12345',
+          indexPatternId: 'id',
+          metrics: [
+            {
+              field: 'client.bytes',
+              type: 'sum',
+            },
+            {
+              field: 'server.bytes',
+              type: 'sum',
+            },
+          ],
+          sourceGeoField: 'client.geo.location',
+          type: 'ES_PEW_PEW',
+        },
+        style: {
+          isTimeAware: true,
+          properties: {
+            fillColor: {
+              options: {
+                color: '#54B399',
+              },
+              type: 'STATIC',
+            },
+            icon: {
+              options: {
+                value: 'marker',
+              },
+              type: 'STATIC',
+            },
+            iconOrientation: {
+              options: {
+                orientation: 0,
+              },
+              type: 'STATIC',
+            },
+            iconSize: {
+              options: {
+                size: 6,
+              },
+              type: 'STATIC',
+            },
+            labelBorderColor: {
+              options: {
+                color: '#FFFFFF',
+              },
+              type: 'STATIC',
+            },
+            labelBorderSize: {
+              options: {
+                size: 'SMALL',
+              },
+            },
+            labelColor: {
+              options: {
+                color: '#000000',
+              },
+              type: 'STATIC',
+            },
+            labelSize: {
+              options: {
+                size: 14,
+              },
+              type: 'STATIC',
+            },
+            labelText: {
+              options: {
+                value: '',
+              },
+              type: 'STATIC',
+            },
+            lineColor: {
+              options: {
+                color: '#6092C0',
+              },
+              type: 'STATIC',
+            },
+            lineWidth: {
+              options: {
+                field: {
+                  name: 'doc_count',
+                  origin: 'source',
+                },
+                fieldMetaOptions: {
+                  isEnabled: true,
+                  sigma: 3,
+                },
+                maxSize: 8,
+                minSize: 1,
+              },
+              type: 'DYNAMIC',
+            },
+            symbolizeAs: {
+              options: {
+                value: 'circle',
+              },
+            },
+          },
+          type: 'VECTOR',
+        },
+        type: 'VECTOR',
+        visible: true,
+      },
+    ]);
+  });
 });
diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.ts
index b2283196a41dd2..8a40ba63bed0dc 100644
--- a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.ts
+++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/create_layer_descriptors.ts
@@ -35,7 +35,11 @@ const defaultDynamicProperties = getDefaultDynamicProperties();
 const euiVisColorPalette = euiPaletteColorBlind();
 
 function isApmIndex(indexPatternTitle: string) {
-  return minimatch(indexPatternTitle, APM_INDEX_PATTERN_TITLE);
+  return APM_INDEX_PATTERN_TITLE.split(',')
+    .map((pattern) => {
+      return minimatch(indexPatternTitle, pattern);
+    })
+    .some(Boolean);
 }
 
 function getSourceField(indexPatternTitle: string) {

From 1fa33ea751e18be2e27dd07d78142fb17bd77e76 Mon Sep 17 00:00:00 2001
From: Oliver Gupte <ogupte@users.noreply.github.com>
Date: Wed, 14 Jul 2021 01:19:29 -0400
Subject: [PATCH 18/27] [APM] Updates the APM static index pattern regardless
 of fleet migration status (#105471)

* [APM] Updates the APM static index pattern regardless of fleet migration status (#105469)

* Fixed and added more unit tests

* ignores eslint error for property name in mock object
---
 .../components/app/Settings/schema/index.tsx  |  4 +-
 .../create_static_index_pattern.test.ts       | 89 +++++++++++++++----
 .../create_static_index_pattern.ts            | 49 ++++++++--
 x-pack/plugins/apm/server/routes/fleet.ts     | 26 +-----
 .../apm/server/routes/index_pattern.ts        |  6 +-
 5 files changed, 122 insertions(+), 52 deletions(-)

diff --git a/x-pack/plugins/apm/public/components/app/Settings/schema/index.tsx b/x-pack/plugins/apm/public/components/app/Settings/schema/index.tsx
index 5a67ce28e9e1a2..0c95648a1cefc3 100644
--- a/x-pack/plugins/apm/public/components/app/Settings/schema/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/Settings/schema/index.tsx
@@ -142,9 +142,7 @@ async function createCloudApmPackagePolicy(
 ) {
   updateLocalStorage(FETCH_STATUS.LOADING);
   try {
-    const {
-      cloud_apm_package_policy: cloudApmPackagePolicy,
-    } = await callApmApi({
+    const { cloudApmPackagePolicy } = await callApmApi({
       endpoint: 'POST /api/apm/fleet/cloud_apm_package_policy',
       signal: null,
     });
diff --git a/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.test.ts b/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.test.ts
index a5340c1220b443..ef869a0ed6cfa5 100644
--- a/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.test.ts
+++ b/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.test.ts
@@ -13,6 +13,11 @@ import { APMConfig } from '../..';
 
 function getMockSavedObjectsClient() {
   return ({
+    get: jest.fn(() => ({
+      attributes: {
+        title: 'apm-*',
+      },
+    })),
     create: jest.fn(),
   } as unknown) as InternalSavedObjectsClient;
 }
@@ -22,14 +27,12 @@ describe('createStaticIndexPattern', () => {
     const setup = {} as Setup;
 
     const savedObjectsClient = getMockSavedObjectsClient();
-    await createStaticIndexPattern(
+    await createStaticIndexPattern({
       setup,
-      {
-        'xpack.apm.autocreateApmIndexPattern': false,
-      } as APMConfig,
+      config: { 'xpack.apm.autocreateApmIndexPattern': false } as APMConfig,
       savedObjectsClient,
-      'default'
-    );
+      spaceId: 'default',
+    });
     expect(savedObjectsClient.create).not.toHaveBeenCalled();
   });
 
@@ -43,14 +46,12 @@ describe('createStaticIndexPattern', () => {
 
     const savedObjectsClient = getMockSavedObjectsClient();
 
-    await createStaticIndexPattern(
+    await createStaticIndexPattern({
       setup,
-      {
-        'xpack.apm.autocreateApmIndexPattern': true,
-      } as APMConfig,
+      config: { 'xpack.apm.autocreateApmIndexPattern': true } as APMConfig,
       savedObjectsClient,
-      'default'
-    );
+      spaceId: 'default',
+    });
     expect(savedObjectsClient.create).not.toHaveBeenCalled();
   });
 
@@ -64,15 +65,73 @@ describe('createStaticIndexPattern', () => {
 
     const savedObjectsClient = getMockSavedObjectsClient();
 
-    await createStaticIndexPattern(
+    await createStaticIndexPattern({
       setup,
-      {
+      config: { 'xpack.apm.autocreateApmIndexPattern': true } as APMConfig,
+      savedObjectsClient,
+      spaceId: 'default',
+    });
+
+    expect(savedObjectsClient.create).toHaveBeenCalled();
+  });
+
+  it(`should upgrade an index pattern if 'apm_oss.indexPattern' does not match title`, async () => {
+    const setup = {} as Setup;
+
+    // does have APM data
+    jest
+      .spyOn(HistoricalAgentData, 'hasHistoricalAgentData')
+      .mockResolvedValue(true);
+
+    const savedObjectsClient = getMockSavedObjectsClient();
+    const apmIndexPatternTitle = 'traces-apm*,logs-apm*,metrics-apm*,apm-*';
+
+    await createStaticIndexPattern({
+      setup,
+      config: {
         'xpack.apm.autocreateApmIndexPattern': true,
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        'apm_oss.indexPattern': apmIndexPatternTitle,
       } as APMConfig,
       savedObjectsClient,
-      'default'
+      spaceId: 'default',
+    });
+
+    expect(savedObjectsClient.get).toHaveBeenCalled();
+    expect(savedObjectsClient.create).toHaveBeenCalled();
+    // @ts-ignore
+    expect(savedObjectsClient.create.mock.calls[0][1].title).toBe(
+      apmIndexPatternTitle
     );
+    // @ts-ignore
+    expect(savedObjectsClient.create.mock.calls[0][2].overwrite).toBe(true);
+  });
+
+  it(`should not upgrade an index pattern if 'apm_oss.indexPattern' already match existing title`, async () => {
+    const setup = {} as Setup;
+
+    // does have APM data
+    jest
+      .spyOn(HistoricalAgentData, 'hasHistoricalAgentData')
+      .mockResolvedValue(true);
+
+    const savedObjectsClient = getMockSavedObjectsClient();
+    const apmIndexPatternTitle = 'apm-*';
+
+    await createStaticIndexPattern({
+      setup,
+      config: {
+        'xpack.apm.autocreateApmIndexPattern': true,
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        'apm_oss.indexPattern': apmIndexPatternTitle,
+      } as APMConfig,
+      savedObjectsClient,
+      spaceId: 'default',
+    });
 
+    expect(savedObjectsClient.get).toHaveBeenCalled();
     expect(savedObjectsClient.create).toHaveBeenCalled();
+    // @ts-ignore
+    expect(savedObjectsClient.create.mock.calls[0][2].overwrite).toBe(false);
   });
 });
diff --git a/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.ts b/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.ts
index a2944d6241d2d9..5dbee59b4ce866 100644
--- a/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.ts
+++ b/x-pack/plugins/apm/server/lib/index_pattern/create_static_index_pattern.ts
@@ -15,13 +15,23 @@ import { InternalSavedObjectsClient } from '../helpers/get_internal_saved_object
 import { withApmSpan } from '../../utils/with_apm_span';
 import { getApmIndexPatternTitle } from './get_apm_index_pattern_title';
 
-export async function createStaticIndexPattern(
-  setup: Setup,
-  config: APMRouteHandlerResources['config'],
-  savedObjectsClient: InternalSavedObjectsClient,
-  spaceId: string | undefined,
-  overwrite = false
-): Promise<boolean> {
+type ApmIndexPatternAttributes = typeof apmIndexPattern.attributes & {
+  title: string;
+};
+
+export async function createStaticIndexPattern({
+  setup,
+  config,
+  savedObjectsClient,
+  spaceId,
+  overwrite = false,
+}: {
+  setup: Setup;
+  config: APMRouteHandlerResources['config'];
+  savedObjectsClient: InternalSavedObjectsClient;
+  spaceId?: string;
+  overwrite?: boolean;
+}): Promise<boolean> {
   return withApmSpan('create_static_index_pattern', async () => {
     // don't autocreate APM index pattern if it's been disabled via the config
     if (!config['xpack.apm.autocreateApmIndexPattern']) {
@@ -35,8 +45,31 @@ export async function createStaticIndexPattern(
       return false;
     }
 
+    const apmIndexPatternTitle = getApmIndexPatternTitle(config);
+
+    if (!overwrite) {
+      try {
+        const {
+          attributes: { title: existingApmIndexPatternTitle },
+        }: {
+          attributes: ApmIndexPatternAttributes;
+        } = await savedObjectsClient.get(
+          'index-pattern',
+          APM_STATIC_INDEX_PATTERN_ID
+        );
+        // if the existing index pattern does not matches the new one, force an update
+        if (existingApmIndexPatternTitle !== apmIndexPatternTitle) {
+          overwrite = true;
+        }
+      } catch (e) {
+        // if the index pattern (saved object) is not found, then we can continue with creation
+        if (!SavedObjectsErrorHelpers.isNotFoundError(e)) {
+          throw e;
+        }
+      }
+    }
+
     try {
-      const apmIndexPatternTitle = getApmIndexPatternTitle(config);
       await withApmSpan('create_index_pattern_saved_object', () =>
         savedObjectsClient.create(
           'index-pattern',
diff --git a/x-pack/plugins/apm/server/routes/fleet.ts b/x-pack/plugins/apm/server/routes/fleet.ts
index b760014d6af89f..66843f4f0df4d7 100644
--- a/x-pack/plugins/apm/server/routes/fleet.ts
+++ b/x-pack/plugins/apm/server/routes/fleet.ts
@@ -25,8 +25,6 @@ import { createCloudApmPackgePolicy } from '../lib/fleet/create_cloud_apm_packag
 import { getUnsupportedApmServerSchema } from '../lib/fleet/get_unsupported_apm_server_schema';
 import { isSuperuser } from '../lib/fleet/is_superuser';
 import { getInternalSavedObjectsClient } from '../lib/helpers/get_internal_saved_objects_client';
-import { setupRequest } from '../lib/helpers/setup_request';
-import { createStaticIndexPattern } from '../lib/index_pattern/create_static_index_pattern';
 
 const hasFleetDataRoute = createApmServerRoute({
   endpoint: 'GET /api/apm/fleet/has_data',
@@ -156,7 +154,7 @@ const createCloudApmPackagePolicyRoute = createApmServerRoute({
   endpoint: 'POST /api/apm/fleet/cloud_apm_package_policy',
   options: { tags: ['access:apm', 'access:apm_write'] },
   handler: async (resources) => {
-    const { plugins, context, config, request, logger, core } = resources;
+    const { plugins, context, config, request, logger } = resources;
     const cloudApmMigrationEnabled =
       config['xpack.apm.agent.migrations.enabled'];
     if (!plugins.fleet || !plugins.security) {
@@ -174,7 +172,7 @@ const createCloudApmPackagePolicyRoute = createApmServerRoute({
       throw Boom.forbidden(CLOUD_SUPERUSER_REQUIRED_MESSAGE);
     }
 
-    const cloudApmAackagePolicy = await createCloudApmPackgePolicy({
+    const cloudApmPackagePolicy = await createCloudApmPackgePolicy({
       cloudPluginSetup,
       fleetPluginStart,
       savedObjectsClient,
@@ -182,25 +180,7 @@ const createCloudApmPackagePolicyRoute = createApmServerRoute({
       logger,
     });
 
-    const [setup, internalSavedObjectsClient] = await Promise.all([
-      setupRequest(resources),
-      core
-        .start()
-        .then(({ savedObjects }) => savedObjects.createInternalRepository()),
-    ]);
-
-    const spaceId = plugins.spaces?.setup.spacesService.getSpaceId(request);
-
-    // force update the index pattern title with data streams
-    await createStaticIndexPattern(
-      setup,
-      config,
-      internalSavedObjectsClient,
-      spaceId,
-      true
-    );
-
-    return { cloud_apm_package_policy: cloudApmAackagePolicy };
+    return { cloudApmPackagePolicy };
   },
 });
 
diff --git a/x-pack/plugins/apm/server/routes/index_pattern.ts b/x-pack/plugins/apm/server/routes/index_pattern.ts
index aa70cde4f96ae2..190baf3bbc2701 100644
--- a/x-pack/plugins/apm/server/routes/index_pattern.ts
+++ b/x-pack/plugins/apm/server/routes/index_pattern.ts
@@ -32,12 +32,12 @@ const staticIndexPatternRoute = createApmServerRoute({
 
     const spaceId = spaces?.setup.spacesService.getSpaceId(request);
 
-    const didCreateIndexPattern = await createStaticIndexPattern(
+    const didCreateIndexPattern = await createStaticIndexPattern({
       setup,
       config,
       savedObjectsClient,
-      spaceId
-    );
+      spaceId,
+    });
 
     return { created: didCreateIndexPattern };
   },

From 604b4242124cd9f0f2612ba90e9490867492f350 Mon Sep 17 00:00:00 2001
From: Christos Nasikas <christos.nasikas@elastic.co>
Date: Wed, 14 Jul 2021 11:21:58 +0300
Subject: [PATCH 19/27] [Actions] Swimlane: Change API Token field to a
 password textfield (#105475)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../swimlane/steps/swimlane_connection.tsx           | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/swimlane/steps/swimlane_connection.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/swimlane/steps/swimlane_connection.tsx
index 2bf99ec9d62b68..e81b607f559718 100644
--- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/swimlane/steps/swimlane_connection.tsx
+++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/swimlane/steps/swimlane_connection.tsx
@@ -4,7 +4,15 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { EuiCallOut, EuiFieldText, EuiFormRow, EuiLink, EuiSpacer, EuiText } from '@elastic/eui';
+import {
+  EuiCallOut,
+  EuiFieldText,
+  EuiFormRow,
+  EuiLink,
+  EuiSpacer,
+  EuiText,
+  EuiFieldPassword,
+} from '@elastic/eui';
 import React, { useCallback } from 'react';
 import { FormattedMessage } from '@kbn/i18n/react';
 import * as i18n from '../translations';
@@ -141,7 +149,7 @@ const SwimlaneConnectionComponent: React.FunctionComponent<Props> = ({
               <EuiSpacer size="m" />
             </>
           )}
-          <EuiFieldText
+          <EuiFieldPassword
             fullWidth
             isInvalid={isApiTokenInvalid}
             readOnly={readOnly}

From b2a6a9d30824f4274749528f4fd1ada94f76df5b Mon Sep 17 00:00:00 2001
From: Angela Chuang <6295984+angorayc@users.noreply.github.com>
Date: Wed, 14 Jul 2021 09:36:56 +0100
Subject: [PATCH 20/27] [SecuritySolution] disable edit and create from rules
 breadcrumb (#105412)

* disable edit and create from rules breadcrumb

* remove href for create /rule breadcrumb

* fix lint error
---
 .../components/navigation/breadcrumbs/index.test.ts  |  5 ++---
 .../detections/pages/detection_engine/rules/utils.ts | 12 ++----------
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/breadcrumbs/index.test.ts b/x-pack/plugins/security_solution/public/common/components/navigation/breadcrumbs/index.test.ts
index 1f7e668b21b988..f415dc287ca35b 100644
--- a/x-pack/plugins/security_solution/public/common/components/navigation/breadcrumbs/index.test.ts
+++ b/x-pack/plugins/security_solution/public/common/components/navigation/breadcrumbs/index.test.ts
@@ -323,8 +323,7 @@ describe('Navigation Breadcrumbs', () => {
         },
         {
           text: 'Create',
-          href:
-            "securitySolution/rules/create?sourcerer=()&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-05-16T23:10:43.696Z',fromStr:now-24h,kind:relative,to:'2019-05-17T23:10:43.697Z',toStr:now)),timeline:(linkTo:!(global),timerange:(from:'2019-05-16T23:10:43.696Z',fromStr:now-24h,kind:relative,to:'2019-05-17T23:10:43.697Z',toStr:now)))",
+          href: '',
         },
       ]);
     });
@@ -382,7 +381,7 @@ describe('Navigation Breadcrumbs', () => {
         },
         {
           text: 'Edit',
-          href: `securitySolution/rules/id/${mockDetailName}/edit?sourcerer=()&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-05-16T23:10:43.696Z',fromStr:now-24h,kind:relative,to:'2019-05-17T23:10:43.697Z',toStr:now)),timeline:(linkTo:!(global),timerange:(from:'2019-05-16T23:10:43.696Z',fromStr:now-24h,kind:relative,to:'2019-05-17T23:10:43.697Z',toStr:now)))`,
+          href: '',
         },
       ]);
     });
diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts
index bbc085eaa0be88..92c828b6cbf79c 100644
--- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts
+++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts
@@ -11,8 +11,6 @@ import { ChromeBreadcrumb } from '../../../../../../../../src/core/public';
 import {
   getRulesUrl,
   getRuleDetailsUrl,
-  getCreateRuleUrl,
-  getEditRuleUrl,
 } from '../../../../common/components/link_to/redirect_to_detection_engine';
 import * as i18nRules from './translations';
 import { RouteSpyState } from '../../../../common/utils/route/types';
@@ -79,10 +77,7 @@ export const getBreadcrumbs = (
       ...breadcrumb,
       {
         text: i18nRules.ADD_PAGE_TITLE,
-        href: getUrlForApp(APP_ID, {
-          deepLinkId: SecurityPageName.rules,
-          path: getCreateRuleUrl(!isEmpty(search[0]) ? search[0] : ''),
-        }),
+        href: '',
       },
     ];
   }
@@ -92,10 +87,7 @@ export const getBreadcrumbs = (
       ...breadcrumb,
       {
         text: i18nRules.EDIT_PAGE_TITLE,
-        href: getUrlForApp(APP_ID, {
-          deepLinkId: SecurityPageName.rules,
-          path: getEditRuleUrl(params.detailName, !isEmpty(search[0]) ? search[0] : ''),
-        }),
+        href: '',
       },
     ];
   }

From 1fe4135ab6ebe039174bb9be2073241867213a2a Mon Sep 17 00:00:00 2001
From: Matthias Wilhelm <matthias.wilhelm@elastic.co>
Date: Wed, 14 Jul 2021 12:18:21 +0200
Subject: [PATCH 21/27] [Discover][Main] Fix missing error message when
 building search query throws exceptions (#103923)

* Fix missing error message when building search fails

* Fix test

* Update _date_nested.ts

* Lint config.js

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../apps/main/services/use_saved_search.ts    | 10 +++++-
 test/functional/apps/discover/_date_nested.ts | 35 +++++++++++++++++++
 test/functional/apps/discover/index.ts        |  1 +
 test/functional/config.js                     | 15 ++++++++
 .../es_archiver/date_nested/data.json         | 30 ++++++++++++++++
 .../es_archiver/date_nested/mappings.json     | 22 ++++++++++++
 6 files changed, 112 insertions(+), 1 deletion(-)
 create mode 100644 test/functional/apps/discover/_date_nested.ts
 create mode 100644 test/functional/fixtures/es_archiver/date_nested/data.json
 create mode 100644 test/functional/fixtures/es_archiver/date_nested/mappings.json

diff --git a/src/plugins/discover/public/application/apps/main/services/use_saved_search.ts b/src/plugins/discover/public/application/apps/main/services/use_saved_search.ts
index 8c847b54078eb7..b449d35dca9ad6 100644
--- a/src/plugins/discover/public/application/apps/main/services/use_saved_search.ts
+++ b/src/plugins/discover/public/application/apps/main/services/use_saved_search.ts
@@ -279,13 +279,21 @@ export const useSavedSearch = ({
     ).pipe(debounceTime(100));
 
     const subscription = fetch$.subscribe((val) => {
-      fetchAll(val === 'reset');
+      try {
+        fetchAll(val === 'reset');
+      } catch (error) {
+        data$.next({
+          state: FetchStatus.ERROR,
+          fetchError: error,
+        });
+      }
     });
 
     return () => {
       subscription.unsubscribe();
     };
   }, [
+    data$,
     data.query.queryString,
     filterManager,
     refetch$,
diff --git a/test/functional/apps/discover/_date_nested.ts b/test/functional/apps/discover/_date_nested.ts
new file mode 100644
index 00000000000000..8297d84832ff64
--- /dev/null
+++ b/test/functional/apps/discover/_date_nested.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const testSubjects = getService('testSubjects');
+  const PageObjects = getPageObjects(['common', 'timePicker', 'discover']);
+  const security = getService('security');
+
+  describe('timefield is a date in a nested field', function () {
+    before(async function () {
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/date_nested');
+      await security.testUser.setRoles(['kibana_admin', 'kibana_date_nested']);
+      await PageObjects.common.navigateToApp('discover');
+    });
+
+    after(async function unloadMakelogs() {
+      await security.testUser.restoreDefaults();
+      await esArchiver.unload('test/functional/fixtures/es_archiver/date_nested');
+    });
+
+    it('should show an error message', async function () {
+      await PageObjects.discover.selectIndexPattern('date-nested');
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+      await testSubjects.existOrFail('discoverNoResultsError');
+    });
+  });
+}
diff --git a/test/functional/apps/discover/index.ts b/test/functional/apps/discover/index.ts
index a17bf53e7f4781..ac8aa50085f33a 100644
--- a/test/functional/apps/discover/index.ts
+++ b/test/functional/apps/discover/index.ts
@@ -51,5 +51,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./_indexpattern_with_unmapped_fields'));
     loadTestFile(require.resolve('./_runtime_fields_editor'));
     loadTestFile(require.resolve('./_huge_fields'));
+    loadTestFile(require.resolve('./_date_nested'));
   });
 }
diff --git a/test/functional/config.js b/test/functional/config.js
index c2c856517c58e4..1c0c519f21e4c8 100644
--- a/test/functional/config.js
+++ b/test/functional/config.js
@@ -247,6 +247,21 @@ export default async function ({ readConfigFile }) {
           },
           kibana: [],
         },
+
+        kibana_date_nested: {
+          elasticsearch: {
+            cluster: [],
+            indices: [
+              {
+                names: ['date-nested'],
+                privileges: ['read', 'view_index_metadata'],
+                field_security: { grant: ['*'], except: [] },
+              },
+            ],
+            run_as: [],
+          },
+          kibana: [],
+        },
         kibana_message_with_newline: {
           elasticsearch: {
             cluster: [],
diff --git a/test/functional/fixtures/es_archiver/date_nested/data.json b/test/functional/fixtures/es_archiver/date_nested/data.json
new file mode 100644
index 00000000000000..0bdb3fc510a63a
--- /dev/null
+++ b/test/functional/fixtures/es_archiver/date_nested/data.json
@@ -0,0 +1,30 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "index-pattern:date-nested",
+    "index": ".kibana",
+    "source": {
+      "index-pattern": {
+       "fields":"[]",
+        "timeFieldName": "@timestamp",
+        "title": "date-nested"
+      },
+      "type": "index-pattern"
+    }
+  }
+}
+
+
+{
+  "type": "doc",
+  "value": {
+    "id": "date-nested-1",
+    "index": "date-nested",
+    "source": {
+      "message" : "test",
+      "nested": {
+        "timestamp": "2021-06-30T12:00:00.123Z"
+      }
+    }
+  }
+}
diff --git a/test/functional/fixtures/es_archiver/date_nested/mappings.json b/test/functional/fixtures/es_archiver/date_nested/mappings.json
new file mode 100644
index 00000000000000..f30e5863f4f8b8
--- /dev/null
+++ b/test/functional/fixtures/es_archiver/date_nested/mappings.json
@@ -0,0 +1,22 @@
+{
+  "type": "index",
+  "value": {
+    "index": "date-nested",
+    "mappings": {
+      "properties": {
+        "message": {
+          "type": "text"
+        },
+        "nested": {
+          "type": "nested"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

From 8fb5633d3658e0ed2c84494bbb09b1ef14da8bd4 Mon Sep 17 00:00:00 2001
From: Jean-Louis Leysens <jloleysens@gmail.com>
Date: Wed, 14 Jul 2021 13:26:34 +0200
Subject: [PATCH 22/27] [Reporting] Fix flaky download pdf test (#105210)

* added a retry for checking that we are on the dashboard landing page

* added .only and removed .skip

* remove .only

* revert this: added .only for flaky test runner

* Remove .only

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 x-pack/test/functional/apps/reporting/reporting.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/x-pack/test/functional/apps/reporting/reporting.ts b/x-pack/test/functional/apps/reporting/reporting.ts
index 9896e3371a2822..8a0d9937fc213e 100644
--- a/x-pack/test/functional/apps/reporting/reporting.ts
+++ b/x-pack/test/functional/apps/reporting/reporting.ts
@@ -12,9 +12,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const pageObjects = getPageObjects(['dashboard', 'common', 'reporting']);
   const es = getService('es');
   const esArchiver = getService('esArchiver');
+  const retry = getService('retry');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/102722
-  describe.skip('Reporting', function () {
+  describe('Reporting', function () {
     this.tags(['smoke', 'ciGroup2']);
     before(async () => {
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/packaging');
@@ -33,6 +33,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       this.timeout(180000);
 
       await pageObjects.common.navigateToApp('dashboards');
+      await retry.waitFor('dashboard landing page', async () => {
+        return await pageObjects.dashboard.onDashboardLandingPage();
+      });
       await pageObjects.dashboard.loadSavedDashboard('dashboard');
       await pageObjects.reporting.openPdfReportingPanel();
       await pageObjects.reporting.clickGenerateReportButton();

From 750c73817421e03fb6ab9065b861c43ce6523ae5 Mon Sep 17 00:00:00 2001
From: Pierre Gayvallet <pierre.gayvallet@gmail.com>
Date: Wed, 14 Jul 2021 13:43:25 +0200
Subject: [PATCH 23/27] fix flaky file_hash unit test (#105447)

* fix flaky unit test

* lint

* uncomment the suite
---
 src/core/server/core_app/bundle_routes/file_hash.test.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/core/server/core_app/bundle_routes/file_hash.test.ts b/src/core/server/core_app/bundle_routes/file_hash.test.ts
index ef24ebe0630572..0de63d7409ed94 100644
--- a/src/core/server/core_app/bundle_routes/file_hash.test.ts
+++ b/src/core/server/core_app/bundle_routes/file_hash.test.ts
@@ -19,8 +19,7 @@ const mockedCache = (): jest.Mocked<IFileHashCache> => ({
   set: jest.fn(),
 });
 
-// FLAKY: https://github.com/elastic/kibana/issues/105174
-describe.skip('getFileHash', () => {
+describe('getFileHash', () => {
   const sampleFilePath = resolve(__dirname, 'foo.js');
   const fd = 42;
   const stats: Stats = { ino: 42, size: 9000 } as any;
@@ -68,6 +67,8 @@ describe.skip('getFileHash', () => {
     await getFileHash(cache, sampleFilePath, stats, fd);
 
     expect(cache.set).toHaveBeenCalledTimes(1);
-    expect(cache.set).toHaveBeenCalledWith(`${sampleFilePath}-${stats.ino}`, computedHashPromise);
+    expect(cache.set).toHaveBeenCalledWith(`${sampleFilePath}-${stats.ino}`, expect.any(Promise));
+    const promiseValue = await cache.set.mock.calls[0][1];
+    expect(promiseValue).toEqual('computed-hash');
   });
 });

From b4fe6b43ddefe8e11f8de2a4346962fec2fff889 Mon Sep 17 00:00:00 2001
From: Maja Grubic <maja.grubic@elastic.co>
Date: Wed, 14 Jul 2021 13:44:07 +0200
Subject: [PATCH 24/27] [Discover] Fix multi-field display when parent field is
 not indexed (#102938)

* Show multifields when parent is not indexed

* [Discover] Fix for multi-fields when parent is not indexed

* Readd package.json

* Applying Tims suggestion

* Add a unit test

* Updating unit test so that it tests the right thing
---
 .../components/table/table.test.tsx           | 32 +++++++++++++++++++
 .../application/components/table/table.tsx    | 16 ++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/src/plugins/discover/public/application/components/table/table.test.tsx b/src/plugins/discover/public/application/components/table/table.test.tsx
index a38a9e41aa242f..5a8d3e7d2db468 100644
--- a/src/plugins/discover/public/application/components/table/table.test.tsx
+++ b/src/plugins/discover/public/application/components/table/table.test.tsx
@@ -334,6 +334,33 @@ describe('DocViewTable at Discover Doc with Fields API', () => {
             },
           },
         },
+        {
+          name: 'city',
+          displayName: 'city',
+          type: 'keyword',
+          isMapped: true,
+          readFromDocValues: true,
+          searchable: true,
+          shortDotsEnable: false,
+          scripted: false,
+          filterable: false,
+        },
+        {
+          name: 'city.raw',
+          displayName: 'city.raw',
+          type: 'string',
+          isMapped: true,
+          spec: {
+            subType: {
+              multi: {
+                parent: 'city',
+              },
+            },
+          },
+          shortDotsEnable: false,
+          scripted: false,
+          filterable: false,
+        },
       ],
     },
     metaFields: ['_index', '_type', '_score', '_id'],
@@ -380,6 +407,7 @@ describe('DocViewTable at Discover Doc with Fields API', () => {
       customer_first_name: 'Betty',
       'customer_first_name.keyword': 'Betty',
       'customer_first_name.nickname': 'Betsy',
+      'city.raw': 'Los Angeles',
     },
   };
   const props = {
@@ -417,6 +445,8 @@ describe('DocViewTable at Discover Doc with Fields API', () => {
       findTestSubject(component, 'tableDocViewRow-customer_first_name.nickname-multifieldBadge')
         .length
     ).toBe(1);
+
+    expect(findTestSubject(component, 'tableDocViewRow-city.raw').length).toBe(1);
   });
 
   it('does not render multifield rows if showMultiFields flag is not set', () => {
@@ -449,5 +479,7 @@ describe('DocViewTable at Discover Doc with Fields API', () => {
       findTestSubject(component, 'tableDocViewRow-customer_first_name.nickname-multifieldBadge')
         .length
     ).toBe(0);
+
+    expect(findTestSubject(component, 'tableDocViewRow-city.raw').length).toBe(1);
   });
 });
diff --git a/src/plugins/discover/public/application/components/table/table.tsx b/src/plugins/discover/public/application/components/table/table.tsx
index 065b146105a651..2d261805d8eb84 100644
--- a/src/plugins/discover/public/application/components/table/table.tsx
+++ b/src/plugins/discover/public/application/components/table/table.tsx
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import React, { useCallback, useMemo } from 'react';
+import React, { useCallback, useMemo, useState } from 'react';
 import { EuiInMemoryTable } from '@elastic/eui';
 import { IndexPattern, IndexPatternField } from '../../../../../data/public';
 import { SHOW_MULTIFIELDS } from '../../../../common';
@@ -60,6 +60,8 @@ export const DocViewerTable = ({
     indexPattern?.fields,
   ]);
 
+  const [childParentFieldsMap] = useState({} as Record<string, string>);
+
   const formattedHit = useMemo(() => indexPattern?.formatHit(hit, 'html'), [hit, indexPattern]);
 
   const tableColumns = useMemo(() => {
@@ -92,11 +94,21 @@ export const DocViewerTable = ({
   }
 
   const flattened = indexPattern.flattenHit(hit);
+  Object.keys(flattened).forEach((key) => {
+    const field = mapping(key);
+    if (field && field.spec?.subType?.multi?.parent) {
+      childParentFieldsMap[field.name] = field.spec.subType.multi.parent;
+    }
+  });
   const items: FieldRecord[] = Object.keys(flattened)
     .filter((fieldName) => {
       const fieldMapping = mapping(fieldName);
       const isMultiField = !!fieldMapping?.spec?.subType?.multi;
-      return isMultiField ? showMultiFields : true;
+      if (!isMultiField) {
+        return true;
+      }
+      const parent = childParentFieldsMap[fieldName];
+      return showMultiFields || (parent && !flattened.hasOwnProperty(parent));
     })
     .sort((fieldA, fieldB) => {
       const mappingA = mapping(fieldA);

From 320eaeac8bdc4ddf928ea35174522980d8dda6bb Mon Sep 17 00:00:00 2001
From: Matthew Kime <matt@mattki.me>
Date: Wed, 14 Jul 2021 07:09:07 -0500
Subject: [PATCH 25/27] Rollup index pattern list fixes (#105458)

* fix index pattern field list and improve typescript
---
 ...in-plugins-data-public.indexpatterntype.md |  19 +++
 ...lugins-data-public.indexpatterntypemeta.md |   1 +
 ...data-public.indexpatterntypemeta.params.md |  13 ++
 .../kibana-plugin-plugins-data-public.md      |   1 +
 .../data/common/index_patterns/types.ts       |  10 +-
 src/plugins/data/public/index.ts              |   1 +
 src/plugins/data/public/public.api.md         |  44 +++--
 .../indexed_fields_table.test.tsx.snap        | 157 ++++++++++++++++++
 .../components/table/table.test.tsx           |   4 +-
 .../indexed_fields_table.test.tsx             |  57 ++++++-
 .../indexed_fields_table.tsx                  |   4 +-
 .../public/service/list/config.ts             |  14 +-
 .../public/service/list/manager.ts            |  13 +-
 ...p_list_config.js => rollup_list_config.ts} |  22 ++-
 14 files changed, 307 insertions(+), 53 deletions(-)
 create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
 create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
 rename src/plugins/index_pattern_management/public/service/list/{rollup_list_config.js => rollup_list_config.ts} (69%)

diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
new file mode 100644
index 00000000000000..46fd3a0725e403
--- /dev/null
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntype.md
@@ -0,0 +1,19 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternType](./kibana-plugin-plugins-data-public.indexpatterntype.md)
+
+## IndexPatternType enum
+
+<b>Signature:</b>
+
+```typescript
+export declare enum IndexPatternType 
+```
+
+## Enumeration Members
+
+|  Member | Value | Description |
+|  --- | --- | --- |
+|  DEFAULT | <code>&quot;default&quot;</code> |  |
+|  ROLLUP | <code>&quot;rollup&quot;</code> |  |
+
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
index e6690b244c9ea5..19a884862d4609 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.md
@@ -15,4 +15,5 @@ export interface TypeMeta
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [aggs](./kibana-plugin-plugins-data-public.indexpatterntypemeta.aggs.md) | <code>Record&lt;string, AggregationRestrictions&gt;</code> |  |
+|  [params](./kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md) | <code>{</code><br/><code>        rollup_index: string;</code><br/><code>    }</code> |  |
 
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
new file mode 100644
index 00000000000000..12646a39188a07
--- /dev/null
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md
@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternTypeMeta](./kibana-plugin-plugins-data-public.indexpatterntypemeta.md) &gt; [params](./kibana-plugin-plugins-data-public.indexpatterntypemeta.params.md)
+
+## IndexPatternTypeMeta.params property
+
+<b>Signature:</b>
+
+```typescript
+params?: {
+        rollup_index: string;
+    };
+```
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
index 65c4601d5faec9..7c2911875ee054 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md
@@ -31,6 +31,7 @@
 |  --- | --- |
 |  [BUCKET\_TYPES](./kibana-plugin-plugins-data-public.bucket_types.md) |  |
 |  [ES\_FIELD\_TYPES](./kibana-plugin-plugins-data-public.es_field_types.md) | \* |
+|  [IndexPatternType](./kibana-plugin-plugins-data-public.indexpatterntype.md) |  |
 |  [KBN\_FIELD\_TYPES](./kibana-plugin-plugins-data-public.kbn_field_types.md) | \* |
 |  [METRIC\_TYPES](./kibana-plugin-plugins-data-public.metric_types.md) |  |
 |  [QuerySuggestionTypes](./kibana-plugin-plugins-data-public.querysuggestiontypes.md) |  |
diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts
index b03e745df74a6a..58cc6d0478d5e8 100644
--- a/src/plugins/data/common/index_patterns/types.ts
+++ b/src/plugins/data/common/index_patterns/types.ts
@@ -150,9 +150,17 @@ export type AggregationRestrictions = Record<
     time_zone?: string;
   }
 >;
+
 export interface TypeMeta {
   aggs?: Record<string, AggregationRestrictions>;
-  [key: string]: any;
+  params?: {
+    rollup_index: string;
+  };
+}
+
+export enum IndexPatternType {
+  DEFAULT = 'default',
+  ROLLUP = 'rollup',
 }
 
 export type FieldSpecConflictDescriptions = Record<string, string[]>;
diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts
index e9e50ebfaf138c..9af1cbf95d94da 100644
--- a/src/plugins/data/public/index.ts
+++ b/src/plugins/data/public/index.ts
@@ -269,6 +269,7 @@ export {
   IndexPatternLoadExpressionFunctionDefinition,
   fieldList,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
+  IndexPatternType,
 } from '../common';
 
 export { DuplicateIndexPatternError } from '../common/index_patterns/errors';
diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
index b8af7c12d57fcc..6c0ddd000f30aa 100644
--- a/src/plugins/data/public/public.api.md
+++ b/src/plugins/data/public/public.api.md
@@ -1688,14 +1688,26 @@ export class IndexPatternsService {
     updateSavedObject(indexPattern: IndexPattern, saveAttempts?: number, ignoreErrors?: boolean): Promise<void | Error>;
 }
 
+// Warning: (ae-missing-release-tag) "IndexPatternType" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export enum IndexPatternType {
+    // (undocumented)
+    DEFAULT = "default",
+    // (undocumented)
+    ROLLUP = "rollup"
+}
+
 // Warning: (ae-missing-release-tag) "TypeMeta" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
 export interface IndexPatternTypeMeta {
-    // (undocumented)
-    [key: string]: any;
     // (undocumented)
     aggs?: Record<string, IndexPatternAggRestrictions>;
+    // (undocumented)
+    params?: {
+        rollup_index: string;
+    };
 }
 
 // Warning: (ae-missing-release-tag) "injectReferences" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -2766,20 +2778,20 @@ export interface WaitUntilNextSessionCompletesOptions {
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:409:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:411:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:412:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:432:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/index.ts:436:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:410:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:412:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:413:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:425:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:430:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:434:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/index.ts:437:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:5 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/search/session/session_service.ts:62:5 - (ae-forgotten-export) The symbol "UrlGeneratorStateMapping" needs to be exported by the entry point index.d.ts
 
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
index 32a2b936edd004..c03899554ffff4 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap
@@ -1,5 +1,148 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`IndexedFieldsTable IndexedFieldsTable with rollup index pattern should render normally 1`] = `
+<div>
+  <Table
+    deleteField={[Function]}
+    editField={[Function]}
+    indexPattern={
+      Object {
+        "getFormatterForFieldNoDefault": [Function],
+        "getNonScriptedFields": [Function],
+        "type": "rollup",
+        "typeMeta": Object {
+          "aggs": Object {
+            "avg": Object {
+              "amount": Object {
+                "agg": "avg",
+              },
+            },
+            "date_histogram": Object {
+              "timestamp": Object {
+                "agg": "date_histogram",
+                "delay": "30s",
+                "fixed_interval": "30s",
+                "time_zone": "UTC",
+              },
+            },
+            "histogram": Object {
+              "amount": Object {
+                "agg": "histogram",
+                "interval": 5,
+              },
+            },
+            "max": Object {
+              "amount": Object {
+                "agg": "max",
+              },
+            },
+            "min": Object {
+              "amount": Object {
+                "agg": "min",
+              },
+            },
+            "sum": Object {
+              "amount": Object {
+                "agg": "sum",
+              },
+            },
+            "terms": Object {
+              "Elastic": Object {
+                "agg": "terms",
+              },
+            },
+            "value_count": Object {
+              "amount": Object {
+                "agg": "value_count",
+              },
+            },
+          },
+          "params": Object {
+            "rollup-index": "rollup",
+          },
+        },
+      }
+    }
+    items={
+      Array [
+        Object {
+          "displayName": "Elastic",
+          "esTypes": Array [
+            "keyword",
+          ],
+          "excluded": false,
+          "format": "",
+          "hasRuntime": false,
+          "info": Array [
+            "Rollup aggregations:",
+            "terms",
+          ],
+          "isMapped": false,
+          "kbnType": undefined,
+          "name": "Elastic",
+          "searchable": true,
+          "type": "keyword",
+        },
+        Object {
+          "displayName": "timestamp",
+          "esTypes": Array [
+            "date",
+          ],
+          "excluded": false,
+          "format": "",
+          "hasRuntime": false,
+          "info": Array [
+            "Rollup aggregations:",
+            "date_histogram (interval: 30s, delay: 30s, UTC)",
+          ],
+          "isMapped": false,
+          "kbnType": undefined,
+          "name": "timestamp",
+          "type": "date",
+        },
+        Object {
+          "displayName": "conflictingField",
+          "esTypes": Array [
+            "keyword",
+            "long",
+          ],
+          "excluded": false,
+          "format": "",
+          "hasRuntime": false,
+          "info": Array [],
+          "isMapped": false,
+          "kbnType": undefined,
+          "name": "conflictingField",
+          "type": "keyword, long",
+        },
+        Object {
+          "displayName": "amount",
+          "esTypes": Array [
+            "long",
+          ],
+          "excluded": false,
+          "format": "",
+          "hasRuntime": false,
+          "info": Array [
+            "Rollup aggregations:",
+            "histogram (interval: 5)",
+            "avg",
+            "max",
+            "min",
+            "sum",
+            "value_count",
+          ],
+          "isMapped": false,
+          "kbnType": undefined,
+          "name": "amount",
+          "type": "long",
+        },
+      ]
+    }
+  />
+</div>
+`;
+
 exports[`IndexedFieldsTable should filter based on the query bar 1`] = `
 <div>
   <Table
@@ -124,6 +267,20 @@ exports[`IndexedFieldsTable should render normally 1`] = `
           "name": "conflictingField",
           "type": "keyword, long",
         },
+        Object {
+          "displayName": "amount",
+          "esTypes": Array [
+            "long",
+          ],
+          "excluded": false,
+          "format": "",
+          "hasRuntime": false,
+          "info": Array [],
+          "isMapped": false,
+          "kbnType": undefined,
+          "name": "amount",
+          "type": "long",
+        },
       ]
     }
   />
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
index b9957534d8c694..163152b52e80b9 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx
@@ -8,13 +8,13 @@
 
 import React from 'react';
 import { shallow } from 'enzyme';
-import { IIndexPattern } from 'src/plugins/data/public';
+import { IndexPattern } from 'src/plugins/data/public';
 import { IndexedFieldItem } from '../../types';
 import { Table, renderFieldName } from './table';
 
 const indexPattern = {
   timeFieldName: 'timestamp',
-} as IIndexPattern;
+} as IndexPattern;
 
 const items: IndexedFieldItem[] = [
   {
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
index e587ada6695cb4..6d37e8f13d6b39 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx
@@ -8,8 +8,9 @@
 
 import React from 'react';
 import { shallow } from 'enzyme';
-import { IndexPatternField, IndexPattern } from 'src/plugins/data/public';
+import { IndexPatternField, IndexPattern, IndexPatternType } from 'src/plugins/data/public';
 import { IndexedFieldsTable } from './indexed_fields_table';
+import { RollupIndexPatternListConfig } from '../../../service/list';
 
 jest.mock('@elastic/eui', () => ({
   EuiFlexGroup: 'eui-flex-group',
@@ -28,7 +29,8 @@ jest.mock('./components/table', () => ({
 const helpers = {
   editField: (fieldName: string) => {},
   deleteField: (fieldName: string) => {},
-  getFieldInfo: () => [],
+  // getFieldInfo handles non rollups as well
+  getFieldInfo: new RollupIndexPatternListConfig().getFieldInfo,
 };
 
 const indexPattern = ({
@@ -36,6 +38,34 @@ const indexPattern = ({
   getFormatterForFieldNoDefault: () => ({ params: () => ({}) }),
 } as unknown) as IndexPattern;
 
+const rollupIndexPattern = ({
+  type: IndexPatternType.ROLLUP,
+  typeMeta: {
+    params: {
+      'rollup-index': 'rollup',
+    },
+    aggs: {
+      date_histogram: {
+        timestamp: {
+          agg: 'date_histogram',
+          fixed_interval: '30s',
+          delay: '30s',
+          time_zone: 'UTC',
+        },
+      },
+      terms: { Elastic: { agg: 'terms' } },
+      histogram: { amount: { agg: 'histogram', interval: 5 } },
+      avg: { amount: { agg: 'avg' } },
+      max: { amount: { agg: 'max' } },
+      min: { amount: { agg: 'min' } },
+      sum: { amount: { agg: 'sum' } },
+      value_count: { amount: { agg: 'value_count' } },
+    },
+  },
+  getNonScriptedFields: () => fields,
+  getFormatterForFieldNoDefault: () => ({ params: () => ({}) }),
+} as unknown) as IndexPattern;
+
 const mockFieldToIndexPatternField = (
   spec: Record<string, string | string[] | boolean | undefined>
 ) => {
@@ -51,6 +81,7 @@ const fields = [
   },
   { name: 'timestamp', displayName: 'timestamp', esTypes: ['date'] },
   { name: 'conflictingField', displayName: 'conflictingField', esTypes: ['keyword', 'long'] },
+  { name: 'amount', displayName: 'amount', esTypes: ['long'] },
 ].map(mockFieldToIndexPatternField);
 
 describe('IndexedFieldsTable', () => {
@@ -115,4 +146,26 @@ describe('IndexedFieldsTable', () => {
 
     expect(component).toMatchSnapshot();
   });
+
+  describe('IndexedFieldsTable with rollup index pattern', () => {
+    test('should render normally', async () => {
+      const component = shallow(
+        <IndexedFieldsTable
+          fields={fields}
+          indexPattern={rollupIndexPattern}
+          helpers={helpers}
+          fieldWildcardMatcher={() => {
+            return () => false;
+          }}
+          indexedFieldTypeFilter=""
+          fieldFilter=""
+        />
+      );
+
+      await new Promise((resolve) => process.nextTick(resolve));
+      component.update();
+
+      expect(component).toMatchSnapshot();
+    });
+  });
 });
diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
index ee1147997079f4..4e9aeb1874c897 100644
--- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
+++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.tsx
@@ -8,7 +8,7 @@
 
 import React, { Component } from 'react';
 import { createSelector } from 'reselect';
-import { IndexPatternField, IndexPattern, IFieldType } from '../../../../../../plugins/data/public';
+import { IndexPatternField, IndexPattern } from '../../../../../../plugins/data/public';
 import { Table } from './components/table';
 import { IndexedFieldItem } from './types';
 
@@ -20,7 +20,7 @@ interface IndexedFieldsTableProps {
   helpers: {
     editField: (fieldName: string) => void;
     deleteField: (fieldName: string) => void;
-    getFieldInfo: (indexPattern: IndexPattern, field: IFieldType) => string[];
+    getFieldInfo: (indexPattern: IndexPattern, field: IndexPatternField) => string[];
   };
   fieldWildcardMatcher: (filters: any[]) => (val: any) => boolean;
 }
diff --git a/src/plugins/index_pattern_management/public/service/list/config.ts b/src/plugins/index_pattern_management/public/service/list/config.ts
index e13f8c1c06241e..4be27fc47d0db8 100644
--- a/src/plugins/index_pattern_management/public/service/list/config.ts
+++ b/src/plugins/index_pattern_management/public/service/list/config.ts
@@ -7,8 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { IIndexPattern, IFieldType } from 'src/plugins/data/public';
-import { SimpleSavedObject } from 'src/core/public';
+import { IndexPattern, IndexPatternField, IndexPatternType } from '../../../../data/public';
 
 export interface IndexPatternTag {
   key: string;
@@ -23,12 +22,9 @@ const defaultIndexPatternListName = i18n.translate(
 );
 
 export class IndexPatternListConfig {
-  public readonly key = 'default';
+  public readonly key: IndexPatternType = IndexPatternType.DEFAULT;
 
-  public getIndexPatternTags(
-    indexPattern: IIndexPattern | SimpleSavedObject<IIndexPattern>,
-    isDefault: boolean
-  ): IndexPatternTag[] {
+  public getIndexPatternTags(indexPattern: IndexPattern, isDefault: boolean): IndexPatternTag[] {
     return isDefault
       ? [
           {
@@ -39,11 +35,11 @@ export class IndexPatternListConfig {
       : [];
   }
 
-  public getFieldInfo(indexPattern: IIndexPattern, field: IFieldType): string[] {
+  public getFieldInfo(indexPattern: IndexPattern, field: IndexPatternField): string[] {
     return [];
   }
 
-  public areScriptedFieldsEnabled(indexPattern: IIndexPattern): boolean {
+  public areScriptedFieldsEnabled(indexPattern: IndexPattern): boolean {
     return true;
   }
 }
diff --git a/src/plugins/index_pattern_management/public/service/list/manager.ts b/src/plugins/index_pattern_management/public/service/list/manager.ts
index bdb2d47057f1f2..d9cefbd8001a5d 100644
--- a/src/plugins/index_pattern_management/public/service/list/manager.ts
+++ b/src/plugins/index_pattern_management/public/service/list/manager.ts
@@ -6,13 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { IIndexPattern, IFieldType } from 'src/plugins/data/public';
-import { SimpleSavedObject } from 'src/core/public';
+import { IndexPattern, IndexPatternField } from 'src/plugins/data/public';
 import { once } from 'lodash';
 import { CoreStart } from '../../../../../core/public';
 import { IndexPatternListConfig, IndexPatternTag } from './config';
 import { CONFIG_ROLLUPS } from '../../constants';
-// @ts-ignore
 import { RollupIndexPatternListConfig } from './rollup_list_config';
 
 interface IndexPatternListManagerStart {
@@ -32,10 +30,7 @@ export class IndexPatternListManager {
       return configs;
     });
     return {
-      getIndexPatternTags: (
-        indexPattern: IIndexPattern | SimpleSavedObject<IIndexPattern>,
-        isDefault: boolean
-      ) =>
+      getIndexPatternTags: (indexPattern: IndexPattern, isDefault: boolean) =>
         getConfigs().reduce(
           (tags: IndexPatternTag[], config) =>
             config.getIndexPatternTags
@@ -44,14 +39,14 @@ export class IndexPatternListManager {
           []
         ),
 
-      getFieldInfo: (indexPattern: IIndexPattern, field: IFieldType): string[] =>
+      getFieldInfo: (indexPattern: IndexPattern, field: IndexPatternField): string[] =>
         getConfigs().reduce(
           (info: string[], config) =>
             config.getFieldInfo ? info.concat(config.getFieldInfo(indexPattern, field)) : info,
           []
         ),
 
-      areScriptedFieldsEnabled: (indexPattern: IIndexPattern): boolean =>
+      areScriptedFieldsEnabled: (indexPattern: IndexPattern): boolean =>
         getConfigs().every((config) =>
           config.areScriptedFieldsEnabled ? config.areScriptedFieldsEnabled(indexPattern) : true
         ),
diff --git a/src/plugins/index_pattern_management/public/service/list/rollup_list_config.js b/src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts
similarity index 69%
rename from src/plugins/index_pattern_management/public/service/list/rollup_list_config.js
rename to src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts
index 9a80d5fd0d622b..bb9da0d461701d 100644
--- a/src/plugins/index_pattern_management/public/service/list/rollup_list_config.js
+++ b/src/plugins/index_pattern_management/public/service/list/rollup_list_config.ts
@@ -6,18 +6,17 @@
  * Side Public License, v 1.
  */
 
+import { IndexPattern, IndexPatternField, IndexPatternType } from '../../../../data/public';
 import { IndexPatternListConfig } from '.';
 
-function isRollup(indexPattern) {
-  return (
-    indexPattern.type === 'rollup' || (indexPattern.get && indexPattern.get('type') === 'rollup')
-  );
+function isRollup(indexPattern: IndexPattern) {
+  return indexPattern.type === IndexPatternType.ROLLUP;
 }
 
 export class RollupIndexPatternListConfig extends IndexPatternListConfig {
-  key = 'rollup';
+  key = IndexPatternType.ROLLUP;
 
-  getIndexPatternTags = (indexPattern) => {
+  getIndexPatternTags = (indexPattern: IndexPattern) => {
     return isRollup(indexPattern)
       ? [
           {
@@ -28,13 +27,13 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
       : [];
   };
 
-  getFieldInfo = (indexPattern, field) => {
+  getFieldInfo = (indexPattern: IndexPattern, field: IndexPatternField) => {
     if (!isRollup(indexPattern)) {
       return [];
     }
 
     const allAggs = indexPattern.typeMeta && indexPattern.typeMeta.aggs;
-    const fieldAggs = allAggs && Object.keys(allAggs).filter((agg) => allAggs[agg][field]);
+    const fieldAggs = allAggs && Object.keys(allAggs).filter((agg) => allAggs[agg][field.name]);
 
     if (!fieldAggs || !fieldAggs.length) {
       return [];
@@ -42,13 +41,12 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
 
     return ['Rollup aggregations:'].concat(
       fieldAggs.map((aggName) => {
-        const agg = allAggs[aggName][field];
+        const agg = allAggs![aggName][field.name];
         switch (aggName) {
           case 'date_histogram':
-            return `${aggName} (interval: ${agg.interval}, ${
+            return `${aggName} (interval: ${agg.fixed_interval}, ${
               agg.delay ? `delay: ${agg.delay},` : ''
             } ${agg.time_zone})`;
-            break;
           case 'histogram':
             return `${aggName} (interval: ${agg.interval})`;
           default:
@@ -58,7 +56,7 @@ export class RollupIndexPatternListConfig extends IndexPatternListConfig {
     );
   };
 
-  areScriptedFieldsEnabled = (indexPattern) => {
+  areScriptedFieldsEnabled = (indexPattern: IndexPattern) => {
     return !isRollup(indexPattern);
   };
 }

From e74c8ee0d81149a2da4fc1afeabf6d932f0a34bd Mon Sep 17 00:00:00 2001
From: Nathan Reese <reese.nathan@gmail.com>
Date: Wed, 14 Jul 2021 06:20:51 -0600
Subject: [PATCH 26/27] [maps] show actionable message when term joins have no
 matches (#105161)

* [maps] show actionable message when term joins have no matches

* include message about showing first 10

* fix formatting

* review feedback

* review feedback

* move performInnerJoins into its own function

* lint

* unit test

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../public/actions/data_request_actions.ts    |  15 +-
 .../maps/public/classes/joins/inner_join.ts   |  16 +-
 .../layers/__fixtures__/mock_sync_context.ts  |   2 +
 .../vector_layer/perform_inner_joins.test.ts  | 285 ++++++++++++++++++
 .../vector_layer/perform_inner_joins.ts       | 134 ++++++++
 .../layers/vector_layer/vector_layer.tsx      |  64 +---
 6 files changed, 447 insertions(+), 69 deletions(-)
 create mode 100644 x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.test.ts
 create mode 100644 x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.ts

diff --git a/x-pack/plugins/maps/public/actions/data_request_actions.ts b/x-pack/plugins/maps/public/actions/data_request_actions.ts
index 47350474eef04f..50df95a52c4d45 100644
--- a/x-pack/plugins/maps/public/actions/data_request_actions.ts
+++ b/x-pack/plugins/maps/public/actions/data_request_actions.ts
@@ -55,6 +55,7 @@ export type DataRequestContext = {
   startLoading(dataId: string, requestToken: symbol, requestMeta?: DataMeta): void;
   stopLoading(dataId: string, requestToken: symbol, data: object, resultsMeta?: DataMeta): void;
   onLoadError(dataId: string, requestToken: symbol, errorMessage: string): void;
+  onJoinError(errorMessage: string): void;
   updateSourceData(newData: unknown): void;
   isRequestStillActive(dataId: string, requestToken: symbol): boolean;
   registerCancelCallback(requestToken: symbol, callback: () => void): void;
@@ -121,6 +122,8 @@ function getDataRequestContext(
       dispatch(endDataLoad(layerId, dataId, requestToken, data, meta)),
     onLoadError: (dataId: string, requestToken: symbol, errorMessage: string) =>
       dispatch(onDataLoadError(layerId, dataId, requestToken, errorMessage)),
+    onJoinError: (errorMessage: string) =>
+      dispatch(setLayerDataLoadErrorStatus(layerId, errorMessage)),
     updateSourceData: (newData: object) => {
       dispatch(updateSourceDataRequest(layerId, newData));
     },
@@ -193,13 +196,11 @@ export function syncDataForLayerId(layerId: string | null) {
 }
 
 function setLayerDataLoadErrorStatus(layerId: string, errorMessage: string | null) {
-  return (dispatch: Dispatch) => {
-    dispatch({
-      type: SET_LAYER_ERROR_STATUS,
-      isInErrorState: errorMessage !== null,
-      layerId,
-      errorMessage,
-    });
+  return {
+    type: SET_LAYER_ERROR_STATUS,
+    isInErrorState: errorMessage !== null,
+    layerId,
+    errorMessage,
   };
 }
 
diff --git a/x-pack/plugins/maps/public/classes/joins/inner_join.ts b/x-pack/plugins/maps/public/classes/joins/inner_join.ts
index 988690233d4849..9f6fec576e9e2a 100644
--- a/x-pack/plugins/maps/public/classes/joins/inner_join.ts
+++ b/x-pack/plugins/maps/public/classes/joins/inner_join.ts
@@ -118,17 +118,23 @@ export class InnerJoin {
       });
     }
 
-    const joinKey = feature.properties[this._leftField.getName()];
-    const coercedKey =
-      typeof joinKey === 'undefined' || joinKey === null ? null : joinKey.toString();
-    if (coercedKey !== null && propertiesMap.has(coercedKey)) {
-      Object.assign(feature.properties, propertiesMap.get(coercedKey));
+    const joinKey = this.getJoinKey(feature);
+    if (joinKey !== null && propertiesMap.has(joinKey)) {
+      Object.assign(feature.properties, propertiesMap.get(joinKey));
       return true;
     } else {
       return false;
     }
   }
 
+  getJoinKey(feature: Feature): string | null {
+    const joinKey =
+      feature.properties && this._leftField
+        ? feature.properties[this._leftField.getName()]
+        : undefined;
+    return joinKey === undefined || joinKey === null ? null : joinKey.toString();
+  }
+
   getRightJoinSource(): ITermJoinSource {
     if (!this._rightSource) {
       throw new Error('Cannot get rightSource from InnerJoin with incomplete config');
diff --git a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts
index dd1367605376d7..16aca6760c4d5b 100644
--- a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts
+++ b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts
@@ -16,6 +16,7 @@ export class MockSyncContext implements DataRequestContext {
   registerCancelCallback: (requestToken: symbol, callback: () => void) => void;
   startLoading: (dataId: string, requestToken: symbol, meta: DataMeta) => void;
   stopLoading: (dataId: string, requestToken: symbol, data: object, meta: DataMeta) => void;
+  onJoinError: (errorMessage: string) => void;
   updateSourceData: (newData: unknown) => void;
   forceRefresh: boolean;
 
@@ -38,6 +39,7 @@ export class MockSyncContext implements DataRequestContext {
     this.registerCancelCallback = sinon.spy();
     this.startLoading = sinon.spy();
     this.stopLoading = sinon.spy();
+    this.onJoinError = sinon.spy();
     this.updateSourceData = sinon.spy();
     this.forceRefresh = false;
   }
diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.test.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.test.ts
new file mode 100644
index 00000000000000..aad1b693be79b8
--- /dev/null
+++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.test.ts
@@ -0,0 +1,285 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import sinon from 'sinon';
+import _ from 'lodash';
+import { FeatureCollection } from 'geojson';
+import { ESTermSourceDescriptor } from '../../../../common/descriptor_types';
+import {
+  AGG_TYPE,
+  FEATURE_VISIBLE_PROPERTY_NAME,
+  SOURCE_TYPES,
+} from '../../../../common/constants';
+import { performInnerJoins } from './perform_inner_joins';
+import { InnerJoin } from '../../joins/inner_join';
+import { IVectorSource } from '../../sources/vector_source';
+import { IField } from '../../fields/field';
+
+const LEFT_FIELD = 'leftKey';
+const COUNT_PROPERTY_NAME = '__kbnjoin__count__d3625663-5b34-4d50-a784-0d743f676a0c';
+const featureCollection = {
+  type: 'FeatureCollection',
+  features: [
+    {
+      type: 'Feature',
+      properties: {
+        [FEATURE_VISIBLE_PROPERTY_NAME]: false,
+        [LEFT_FIELD]: 'alpha',
+      },
+      geometry: {
+        type: 'Point',
+        coordinates: [-112, 46],
+      },
+    },
+    {
+      type: 'Feature',
+      properties: {
+        [COUNT_PROPERTY_NAME]: 20,
+        [FEATURE_VISIBLE_PROPERTY_NAME]: true,
+        [LEFT_FIELD]: 'bravo',
+      },
+      geometry: {
+        type: 'Point',
+        coordinates: [-100, 40],
+      },
+    },
+  ],
+};
+
+const joinDescriptor = {
+  leftField: LEFT_FIELD,
+  right: {
+    applyGlobalQuery: true,
+    applyGlobalTime: true,
+    id: 'd3625663-5b34-4d50-a784-0d743f676a0c',
+    indexPatternId: 'myIndexPattern',
+    metrics: [
+      {
+        type: AGG_TYPE.COUNT,
+      },
+    ],
+    term: 'rightKey',
+    type: SOURCE_TYPES.ES_TERM_SOURCE,
+  } as ESTermSourceDescriptor,
+};
+const mockVectorSource = ({
+  getInspectorAdapters: () => {
+    return undefined;
+  },
+  createField: () => {
+    return {
+      getName: () => {
+        return LEFT_FIELD;
+      },
+    } as IField;
+  },
+} as unknown) as IVectorSource;
+const innerJoin = new InnerJoin(joinDescriptor, mockVectorSource);
+const propertiesMap = new Map<string, Record<string | number, unknown>>();
+propertiesMap.set('alpha', { [COUNT_PROPERTY_NAME]: 1 });
+
+test('should skip join when no state has changed', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  performInnerJoins(
+    {
+      refreshed: false,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: false,
+        join: innerJoin,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  expect(updateSourceData.notCalled);
+  expect(onJoinError.notCalled);
+});
+
+test('should perform join when features change', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  performInnerJoins(
+    {
+      refreshed: true,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: false,
+        join: innerJoin,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  expect(updateSourceData.calledOnce);
+  expect(onJoinError.notCalled);
+});
+
+test('should perform join when join state changes', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  performInnerJoins(
+    {
+      refreshed: false,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: true,
+        join: innerJoin,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  expect(updateSourceData.calledOnce);
+  expect(onJoinError.notCalled);
+});
+
+test('should call updateSourceData with feature collection with updated feature visibility and join properties', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  performInnerJoins(
+    {
+      refreshed: true,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: false,
+        join: innerJoin,
+        propertiesMap,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  const firstCallArgs = updateSourceData.args[0];
+  const updateSourceDataFeatureCollection = firstCallArgs[0];
+  expect(updateSourceDataFeatureCollection).toEqual({
+    type: 'FeatureCollection',
+    features: [
+      {
+        type: 'Feature',
+        properties: {
+          [COUNT_PROPERTY_NAME]: 1,
+          [FEATURE_VISIBLE_PROPERTY_NAME]: true,
+          [LEFT_FIELD]: 'alpha',
+        },
+        geometry: {
+          type: 'Point',
+          coordinates: [-112, 46],
+        },
+      },
+      {
+        type: 'Feature',
+        properties: {
+          [FEATURE_VISIBLE_PROPERTY_NAME]: false,
+          [LEFT_FIELD]: 'bravo',
+        },
+        geometry: {
+          type: 'Point',
+          coordinates: [-100, 40],
+        },
+      },
+    ],
+  });
+  expect(onJoinError.notCalled);
+});
+
+test('should call updateSourceData when no results returned from terms aggregation', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  performInnerJoins(
+    {
+      refreshed: false,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: true,
+        join: innerJoin,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  const firstCallArgs = updateSourceData.args[0];
+  const updateSourceDataFeatureCollection = firstCallArgs[0];
+  expect(updateSourceDataFeatureCollection).toEqual({
+    type: 'FeatureCollection',
+    features: [
+      {
+        type: 'Feature',
+        properties: {
+          [FEATURE_VISIBLE_PROPERTY_NAME]: false,
+          [LEFT_FIELD]: 'alpha',
+        },
+        geometry: {
+          type: 'Point',
+          coordinates: [-112, 46],
+        },
+      },
+      {
+        type: 'Feature',
+        properties: {
+          [COUNT_PROPERTY_NAME]: 20,
+          [FEATURE_VISIBLE_PROPERTY_NAME]: false,
+          [LEFT_FIELD]: 'bravo',
+        },
+        geometry: {
+          type: 'Point',
+          coordinates: [-100, 40],
+        },
+      },
+    ],
+  });
+  expect(onJoinError.notCalled);
+});
+
+test('should call onJoinError when there are no matching features', () => {
+  const updateSourceData = sinon.spy();
+  const onJoinError = sinon.spy();
+
+  // instead of returning military alphabet like "alpha" or "bravo", mismatched key returns numbers, like '1'
+  const propertiesMapFromMismatchedKey = new Map<string, Record<string | number, unknown>>();
+  propertiesMapFromMismatchedKey.set('1', { [COUNT_PROPERTY_NAME]: 1 });
+
+  performInnerJoins(
+    {
+      refreshed: false,
+      featureCollection: _.cloneDeep(featureCollection) as FeatureCollection,
+    },
+    [
+      {
+        dataHasChanged: true,
+        join: innerJoin,
+        propertiesMap: propertiesMapFromMismatchedKey,
+      },
+    ],
+    updateSourceData,
+    onJoinError
+  );
+
+  expect(updateSourceData.notCalled);
+  expect(onJoinError.calledOnce);
+});
diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.ts
new file mode 100644
index 00000000000000..23c6527d3e8180
--- /dev/null
+++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/perform_inner_joins.ts
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FeatureCollection } from 'geojson';
+import { i18n } from '@kbn/i18n';
+import { FEATURE_VISIBLE_PROPERTY_NAME } from '../../../../common/constants';
+import { DataRequestContext } from '../../../actions';
+import { InnerJoin } from '../../joins/inner_join';
+import { PropertiesMap } from '../../../../common/elasticsearch_util';
+
+interface SourceResult {
+  refreshed: boolean;
+  featureCollection: FeatureCollection;
+}
+
+export interface JoinState {
+  dataHasChanged: boolean;
+  join: InnerJoin;
+  propertiesMap?: PropertiesMap;
+}
+
+export function performInnerJoins(
+  sourceResult: SourceResult,
+  joinStates: JoinState[],
+  updateSourceData: DataRequestContext['updateSourceData'],
+  onJoinError: DataRequestContext['onJoinError']
+) {
+  // should update the store if
+  // -- source result was refreshed
+  // -- any of the join configurations changed (joinState changed)
+  // -- visibility of any of the features has changed
+
+  let shouldUpdateStore =
+    sourceResult.refreshed || joinStates.some((joinState) => joinState.dataHasChanged);
+
+  if (!shouldUpdateStore) {
+    return;
+  }
+
+  const joinStatuses = joinStates.map((joinState) => {
+    return {
+      joinedWithAtLeastOneFeature: false,
+      keys: [] as string[],
+      joinState,
+    };
+  });
+
+  for (let i = 0; i < sourceResult.featureCollection.features.length; i++) {
+    const feature = sourceResult.featureCollection.features[i];
+    if (!feature.properties) {
+      feature.properties = {};
+    }
+    const oldVisbility = feature.properties[FEATURE_VISIBLE_PROPERTY_NAME];
+    let isFeatureVisible = true;
+    for (let j = 0; j < joinStates.length; j++) {
+      const joinState = joinStates[j];
+      const innerJoin = joinState.join;
+      const joinStatus = joinStatuses[j];
+      const joinKey = innerJoin.getJoinKey(feature);
+      if (joinKey !== null) {
+        joinStatus.keys.push(joinKey);
+      }
+      const canJoinOnCurrent = joinState.propertiesMap
+        ? innerJoin.joinPropertiesToFeature(feature, joinState.propertiesMap)
+        : false;
+      if (canJoinOnCurrent && !joinStatus.joinedWithAtLeastOneFeature) {
+        joinStatus.joinedWithAtLeastOneFeature = true;
+      }
+      isFeatureVisible = isFeatureVisible && canJoinOnCurrent;
+    }
+
+    if (oldVisbility !== isFeatureVisible) {
+      shouldUpdateStore = true;
+    }
+
+    feature.properties[FEATURE_VISIBLE_PROPERTY_NAME] = isFeatureVisible;
+  }
+
+  if (shouldUpdateStore) {
+    updateSourceData({ ...sourceResult.featureCollection });
+  }
+
+  const joinStatusesWithoutAnyMatches = joinStatuses.filter((joinStatus) => {
+    return (
+      !joinStatus.joinedWithAtLeastOneFeature && joinStatus.joinState.propertiesMap !== undefined
+    );
+  });
+
+  if (joinStatusesWithoutAnyMatches.length) {
+    function prettyPrintArray(array: unknown[]) {
+      return array.length <= 5
+        ? array.join(',')
+        : array.slice(0, 5).join(',') +
+            i18n.translate('xpack.maps.vectorLayer.joinError.firstTenMsg', {
+              defaultMessage: ` (5 of {total})`,
+              values: { total: array.length },
+            });
+    }
+
+    const joinStatus = joinStatusesWithoutAnyMatches[0];
+    const leftFieldName = joinStatus.joinState.join.getLeftField().getName();
+    const reason =
+      joinStatus.keys.length === 0
+        ? i18n.translate('xpack.maps.vectorLayer.joinError.noLeftFieldValuesMsg', {
+            defaultMessage: `Left field: '{leftFieldName}', does not provide any values.`,
+            values: { leftFieldName },
+          })
+        : i18n.translate('xpack.maps.vectorLayer.joinError.noMatchesMsg', {
+            defaultMessage: `Left field does not match right field. Left field: '{leftFieldName}' has values { leftFieldValues }. Right field: '{rightFieldName}' has values: { rightFieldValues }.`,
+            values: {
+              leftFieldName,
+              leftFieldValues: prettyPrintArray(joinStatus.keys),
+              rightFieldName: joinStatus.joinState.join
+                .getRightJoinSource()
+                .getTermField()
+                .getName(),
+              rightFieldValues: prettyPrintArray(
+                Array.from(joinStatus.joinState.propertiesMap!.keys())
+              ),
+            },
+          });
+
+    onJoinError(
+      i18n.translate('xpack.maps.vectorLayer.joinErrorMsg', {
+        defaultMessage: `Unable to perform term join. {reason}`,
+        values: { reason },
+      })
+    );
+  }
+}
diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx
index 959064b3daab20..74a0ea1e638823 100644
--- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx
+++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx
@@ -69,17 +69,7 @@ import { IESSource } from '../../sources/es_source';
 import { PropertiesMap } from '../../../../common/elasticsearch_util';
 import { ITermJoinSource } from '../../sources/term_join_source';
 import { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from './utils';
-
-interface SourceResult {
-  refreshed: boolean;
-  featureCollection?: FeatureCollection;
-}
-
-interface JoinState {
-  dataHasChanged: boolean;
-  join: InnerJoin;
-  propertiesMap?: PropertiesMap;
-}
+import { JoinState, performInnerJoins } from './perform_inner_joins';
 
 export interface VectorLayerArguments {
   source: IVectorSource;
@@ -435,51 +425,6 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer {
     };
   }
 
-  _performInnerJoins(
-    sourceResult: SourceResult,
-    joinStates: JoinState[],
-    updateSourceData: DataRequestContext['updateSourceData']
-  ) {
-    // should update the store if
-    // -- source result was refreshed
-    // -- any of the join configurations changed (joinState changed)
-    // -- visibility of any of the features has changed
-
-    let shouldUpdateStore =
-      sourceResult.refreshed || joinStates.some((joinState) => joinState.dataHasChanged);
-
-    if (!shouldUpdateStore) {
-      return;
-    }
-
-    for (let i = 0; i < sourceResult.featureCollection!.features.length; i++) {
-      const feature = sourceResult.featureCollection!.features[i];
-      if (!feature.properties) {
-        feature.properties = {};
-      }
-      const oldVisbility = feature.properties[FEATURE_VISIBLE_PROPERTY_NAME];
-      let isFeatureVisible = true;
-      for (let j = 0; j < joinStates.length; j++) {
-        const joinState = joinStates[j];
-        const innerJoin = joinState.join;
-        const canJoinOnCurrent = joinState.propertiesMap
-          ? innerJoin.joinPropertiesToFeature(feature, joinState.propertiesMap)
-          : false;
-        isFeatureVisible = isFeatureVisible && canJoinOnCurrent;
-      }
-
-      if (oldVisbility !== isFeatureVisible) {
-        shouldUpdateStore = true;
-      }
-
-      feature.properties[FEATURE_VISIBLE_PROPERTY_NAME] = isFeatureVisible;
-    }
-
-    if (shouldUpdateStore) {
-      updateSourceData({ ...sourceResult.featureCollection });
-    }
-  }
-
   async _syncSourceStyleMeta(
     syncContext: DataRequestContext,
     source: IVectorSource,
@@ -714,7 +659,12 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer {
       }
 
       const joinStates = await this._syncJoins(syncContext, style);
-      this._performInnerJoins(sourceResult, joinStates, syncContext.updateSourceData);
+      performInnerJoins(
+        sourceResult,
+        joinStates,
+        syncContext.updateSourceData,
+        syncContext.onJoinError
+      );
     } catch (error) {
       if (!(error instanceof DataRequestAbortError)) {
         throw error;

From 509026d2409e906cd143ea739f83a577d448f26d Mon Sep 17 00:00:00 2001
From: Marco Liberati <dej611@users.noreply.github.com>
Date: Wed, 14 Jul 2021 15:47:41 +0200
Subject: [PATCH 27/27] [Lens][Inspector] Enable inspector to display multiple
 requests for multiple layers (#105224)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 ...in-plugins-expressions-public.execution.md |  2 +-
 ...gins-expressions-public.execution.start.md |  3 +-
 ...in-plugins-expressions-server.execution.md |  2 +-
 ...gins-expressions-server.execution.start.md |  3 +-
 .../expressions/esaggs/request_handler.ts     |  2 -
 .../common/execution/execution.test.ts        |  8 +++
 .../expressions/common/execution/execution.ts |  9 +++-
 src/plugins/expressions/public/public.api.md  |  2 +-
 src/plugins/expressions/server/server.api.md  |  2 +-
 x-pack/test/functional/apps/lens/dashboard.ts | 51 +++++++++++++++++++
 10 files changed, 74 insertions(+), 10 deletions(-)

diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md
index edaf1c9a9ce9eb..040bed5a8ce533 100644
--- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md
+++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md
@@ -39,5 +39,5 @@ export declare class Execution<Input = unknown, Output = unknown, InspectorAdapt
 |  [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-public.execution.invokechain.md) |  |  |
 |  [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-public.execution.invokefunction.md) |  |  |
 |  [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-public.execution.resolveargs.md) |  |  |
-|  [start(input)](./kibana-plugin-plugins-expressions-public.execution.start.md) |  | Call this method to start execution.<!-- -->N.B. <code>input</code> is initialized to <code>null</code> rather than <code>undefined</code> for legacy reasons, because in legacy interpreter it was set to <code>null</code> by default. |
+|  [start(input, isSubExpression)](./kibana-plugin-plugins-expressions-public.execution.start.md) |  | Call this method to start execution.<!-- -->N.B. <code>input</code> is initialized to <code>null</code> rather than <code>undefined</code> for legacy reasons, because in legacy interpreter it was set to <code>null</code> by default. |
 
diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.start.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.start.md
index 352226da6d72ad..b1fa6d7d518b99 100644
--- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.start.md
+++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.start.md
@@ -11,7 +11,7 @@ N.B. `input` is initialized to `null` rather than `undefined` for legacy reasons
 <b>Signature:</b>
 
 ```typescript
-start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>;
+start(input?: Input, isSubExpression?: boolean): Observable<ExecutionResult<Output | ExpressionValueError>>;
 ```
 
 ## Parameters
@@ -19,6 +19,7 @@ start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  input | <code>Input</code> |  |
+|  isSubExpression | <code>boolean</code> |  |
 
 <b>Returns:</b>
 
diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md
index 47963e5e5ef46f..44d16ea02e2708 100644
--- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md
+++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md
@@ -39,5 +39,5 @@ export declare class Execution<Input = unknown, Output = unknown, InspectorAdapt
 |  [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-server.execution.invokechain.md) |  |  |
 |  [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-server.execution.invokefunction.md) |  |  |
 |  [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-server.execution.resolveargs.md) |  |  |
-|  [start(input)](./kibana-plugin-plugins-expressions-server.execution.start.md) |  | Call this method to start execution.<!-- -->N.B. <code>input</code> is initialized to <code>null</code> rather than <code>undefined</code> for legacy reasons, because in legacy interpreter it was set to <code>null</code> by default. |
+|  [start(input, isSubExpression)](./kibana-plugin-plugins-expressions-server.execution.start.md) |  | Call this method to start execution.<!-- -->N.B. <code>input</code> is initialized to <code>null</code> rather than <code>undefined</code> for legacy reasons, because in legacy interpreter it was set to <code>null</code> by default. |
 
diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.start.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.start.md
index 0eef7013cb3c61..23b4d414d09d14 100644
--- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.start.md
+++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.start.md
@@ -11,7 +11,7 @@ N.B. `input` is initialized to `null` rather than `undefined` for legacy reasons
 <b>Signature:</b>
 
 ```typescript
-start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>;
+start(input?: Input, isSubExpression?: boolean): Observable<ExecutionResult<Output | ExpressionValueError>>;
 ```
 
 ## Parameters
@@ -19,6 +19,7 @@ start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  input | <code>Input</code> |  |
+|  isSubExpression | <code>boolean</code> |  |
 
 <b>Returns:</b>
 
diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts
index 61193c52a5e74b..bf931966f5baed 100644
--- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts
+++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts
@@ -100,8 +100,6 @@ export const handleRequest = async ({
   requestSearchSource.setField('filter', filters);
   requestSearchSource.setField('query', query);
 
-  inspectorAdapters.requests?.reset();
-
   const { rawResponse: response } = await requestSearchSource
     .fetch$({
       abortSignal,
diff --git a/src/plugins/expressions/common/execution/execution.test.ts b/src/plugins/expressions/common/execution/execution.test.ts
index 8c6f457105d426..2e9d4b91908a05 100644
--- a/src/plugins/expressions/common/execution/execution.test.ts
+++ b/src/plugins/expressions/common/execution/execution.test.ts
@@ -339,6 +339,14 @@ describe('Execution', () => {
       });
       expect(execution.inspectorAdapters).toBe(inspectorAdapters);
     });
+
+    test('it should reset the request adapter only on startup', async () => {
+      const inspectorAdapters = { requests: { reset: jest.fn() } };
+      await run('add val={add 5 | access "value"}', {
+        inspectorAdapters,
+      });
+      expect(inspectorAdapters.requests.reset).toHaveBeenCalledTimes(1);
+    });
   });
 
   describe('expression abortion', () => {
diff --git a/src/plugins/expressions/common/execution/execution.ts b/src/plugins/expressions/common/execution/execution.ts
index 47209c348257e6..ef925b3a68294c 100644
--- a/src/plugins/expressions/common/execution/execution.ts
+++ b/src/plugins/expressions/common/execution/execution.ts
@@ -280,13 +280,18 @@ export class Execution<
    * because in legacy interpreter it was set to `null` by default.
    */
   public start(
-    input: Input = null as any
+    input: Input = null as any,
+    isSubExpression?: boolean
   ): Observable<ExecutionResult<Output | ExpressionValueError>> {
     if (this.hasStarted) throw new Error('Execution already started.');
     this.hasStarted = true;
     this.input = input;
     this.state.transitions.start();
 
+    if (!isSubExpression) {
+      this.context.inspectorAdapters.requests?.reset();
+    }
+
     if (isObservable<Input>(input)) {
       input.subscribe(this.input$);
     } else if (isPromise(input)) {
@@ -534,7 +539,7 @@ export class Execution<
         );
         this.childExecutions.push(execution);
 
-        return execution.start(input);
+        return execution.start(input, true);
       case 'string':
       case 'number':
       case 'null':
diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md
index 2d9c6d94cfa6dc..55655cfc5d1564 100644
--- a/src/plugins/expressions/public/public.api.md
+++ b/src/plugins/expressions/public/public.api.md
@@ -120,7 +120,7 @@ export class Execution<Input = unknown, Output = unknown, InspectorAdapters exte
     // (undocumented)
     resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Observable<any>;
     readonly result: Observable<ExecutionResult<Output | ExpressionValueError>>;
-    start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>;
+    start(input?: Input, isSubExpression?: boolean): Observable<ExecutionResult<Output | ExpressionValueError>>;
     // Warning: (ae-forgotten-export) The symbol "ExecutionResult" needs to be exported by the entry point index.d.ts
     readonly state: ExecutionContainer<ExecutionResult<Output | ExpressionValueError>>;
 }
diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md
index ec16d95ea8a3ff..a34727e7a770f5 100644
--- a/src/plugins/expressions/server/server.api.md
+++ b/src/plugins/expressions/server/server.api.md
@@ -118,7 +118,7 @@ export class Execution<Input = unknown, Output = unknown, InspectorAdapters exte
     // (undocumented)
     resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Observable<any>;
     readonly result: Observable<ExecutionResult<Output | ExpressionValueError>>;
-    start(input?: Input): Observable<ExecutionResult<Output | ExpressionValueError>>;
+    start(input?: Input, isSubExpression?: boolean): Observable<ExecutionResult<Output | ExpressionValueError>>;
     // Warning: (ae-forgotten-export) The symbol "ExecutionResult" needs to be exported by the entry point index.d.ts
     readonly state: ExecutionContainer<ExecutionResult<Output | ExpressionValueError>>;
 }
diff --git a/x-pack/test/functional/apps/lens/dashboard.ts b/x-pack/test/functional/apps/lens/dashboard.ts
index 6e4c20744c5fcc..095b1ae7c2f0f5 100644
--- a/x-pack/test/functional/apps/lens/dashboard.ts
+++ b/x-pack/test/functional/apps/lens/dashboard.ts
@@ -27,6 +27,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const filterBar = getService('filterBar');
   const security = getService('security');
   const panelActions = getService('dashboardPanelActions');
+  const inspector = getService('inspector');
 
   async function clickInChart(x: number, y: number) {
     const el = await elasticChart.getCanvas();
@@ -158,6 +159,56 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await testSubjects.existOrFail(ACTION_TEST_SUBJ);
     });
 
+    it('should show all data from all layers in the inspector', async () => {
+      await PageObjects.common.navigateToApp('dashboard');
+      await PageObjects.dashboard.clickNewDashboard();
+      await dashboardAddPanel.clickCreateNewLink();
+      await PageObjects.header.waitUntilLoadingHasFinished();
+      await PageObjects.lens.goToTimeRange();
+      await PageObjects.lens.configureDimension({
+        dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension',
+        operation: 'date_histogram',
+        field: '@timestamp',
+      });
+
+      await PageObjects.lens.configureDimension({
+        dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
+        operation: 'average',
+        field: 'bytes',
+      });
+
+      await PageObjects.lens.createLayer();
+
+      expect(await PageObjects.lens.hasChartSwitchWarning('line')).to.eql(false);
+
+      await PageObjects.lens.switchToVisualization('line');
+      await PageObjects.lens.configureDimension(
+        {
+          dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension',
+          operation: 'date_histogram',
+          field: '@timestamp',
+        },
+        1
+      );
+
+      await PageObjects.lens.configureDimension(
+        {
+          dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
+          operation: 'median',
+          field: 'bytes',
+        },
+        1
+      );
+      await PageObjects.lens.saveAndReturn();
+
+      await panelActions.openContextMenu();
+      await panelActions.clickContextMenuMoreItem();
+      await testSubjects.click('embeddablePanelAction-openInspector');
+      await inspector.openInspectorRequestsView();
+      const requests = await inspector.getRequestNames();
+      expect(requests.split(',').length).to.be(2);
+    });
+
     it('unlink lens panel from embeddable library', async () => {
       await PageObjects.common.navigateToApp('dashboard');
       await PageObjects.dashboard.clickNewDashboard();