Merge branch 'master' into alerting/index_threshold_fix

docs/developer/plugin-list.asciidoc

@@ -216,7 +216,7 @@ generating deep links to other apps, and creating short URLs.
 
 
 |{kib-repo}blob/{branch}/src/plugins/telemetry_management_section/README.md[telemetryManagementSection]
-|This plugin adds the Advanced Settings section for the Usage Data collection (aka Telemetry).
+|This plugin adds the Advanced Settings section for the Usage and Security Data collection (aka Telemetry).
 
 
 |{kib-repo}blob/{branch}/src/plugins/tile_map/README.md[tileMap]

docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md

@@ -22,11 +22,12 @@ export interface IExpressionLoaderParams
 |  [hasCompatibleActions](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.hascompatibleactions.md) | <code>ExpressionRenderHandlerParams['hasCompatibleActions']</code> |  |
 |  [inspectorAdapters](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.inspectoradapters.md) | <code>Adapters</code> |  |
 |  [onRenderError](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.onrendererror.md) | <code>RenderErrorHandlerFnType</code> |  |
-|  [partial](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.partial.md) | <code>boolean</code> |  |
+|  [partial](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.partial.md) | <code>boolean</code> | The flag to toggle on emitting partial results. By default, the partial results are disabled. |
 |  [renderMode](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.rendermode.md) | <code>RenderMode</code> |  |
 |  [searchContext](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchcontext.md) | <code>SerializableState</code> |  |
 |  [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) | <code>string</code> |  |
 |  [syncColors](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.synccolors.md) | <code>boolean</code> |  |
+|  [throttle](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.throttle.md) | <code>number</code> | Throttling of partial results in milliseconds. By default, throttling is disabled. |
 |  [uiState](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.uistate.md) | <code>unknown</code> |  |
 |  [variables](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.variables.md) | <code>Record&lt;string, any&gt;</code> |  |
 

docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.partial.md

@@ -4,6 +4,8 @@
 
 ## IExpressionLoaderParams.partial property
 
+The flag to toggle on emitting partial results. By default, the partial results are disabled.
+
 <b>Signature:</b>
 
 ```typescript

docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.throttle.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) &gt; [IExpressionLoaderParams](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md) &gt; [throttle](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.throttle.md)
+
+## IExpressionLoaderParams.throttle property
+
+Throttling of partial results in milliseconds. By default, throttling is disabled.
+
+<b>Signature:</b>
+
+```typescript
+throttle?: number;
+```

docs/settings/security-settings.asciidoc

@@ -337,24 +337,12 @@ For more details and a reference of audit events, refer to <<xpack-security-audi
 [cols="2*<"]
 |======
 | `xpack.security.audit.enabled` {ess-icon}
-| Set to `true` to enable audit logging for security events. *Default:* `false`
-|======
-
-[float]
-[[ecs-audit-logging-settings]]
-==== ECS audit logging settings
-
-To enable the <<xpack-security-ecs-audit-logging, ECS audit logger>>, specify where you want to write the audit events using `xpack.security.audit.appender`.
-
-[cols="2*<,*50"]
-|======
-| `xpack.security.audit.appender`
-| Optional. Specifies where audit logs should be written to and how they should be formatted.
+| Set to `true` _and_ configure an appender with `xpack.security.audit.appender` to enable ECS audit logging`. *Default:* `false`
 
 2+a| For example:
-
 [source,yaml]
 ----------------------------------------
+xpack.security.audit.enabled: true
 xpack.security.audit.appender:
   type: rolling-file
   fileName: ./audit.log
@@ -370,7 +358,31 @@ xpack.security.audit.appender:
 <1> Rotates log files every 24 hours.
 <2> Keeps maximum of 10 log files before deleting older ones.
 
-| `xpack.security.audit.appender.type`
+[NOTE]
+============
+{ess} does not support custom log file policies. To enable audit logging on {ess} only specify:
+
+[source,yaml]
+----------------------------------------
+xpack.security.audit.enabled: true
+xpack.security.audit.appender.type: rolling-file
+----------------------------------------
+============
+
+[NOTE]
+============
+deprecated:[7.15.0,"In 8.0 and later, the legacy audit logger will be removed, and this setting will enable the ECS audit logger with a default appender."] To enable the legacy audit logger only specify:
+
+[source,yaml]
+----------------------------------------
+xpack.security.audit.enabled: true
+----------------------------------------
+============
+
+| `xpack.security.audit.appender` {ess-icon}
+| Optional. Specifies where audit logs should be written to and how they should be formatted.
+
+| `xpack.security.audit.appender.type` {ess-icon}
 | Required. Specifies where audit logs should be written to. Allowed values are `console`, `file`, or `rolling-file`. 
 
 Refer to <<audit-logging-file-appender>> and <<audit-logging-rolling-file-appender>> for appender specific settings.

docs/user/security/audit-logging.asciidoc

@@ -14,16 +14,24 @@ by cluster-wide privileges. For more information on enabling audit logging in
 
 [IMPORTANT]
 ============================================================================
+Kibana offers two audit logs: a **deprecated** legacy audit logger, and a new
+ECS-compliant audit logger. We strongly advise using the <<xpack-security-ecs-audit-logging, ECS audit logger>>,
+as the legacy audit logger will be removed in an upcoming version.
+============================================================================
+
+[NOTE]
+============================================================================
 Audit logs are **disabled** by default. To enable this functionality, you must
-set `xpack.security.audit.enabled` to `true` in `kibana.yml`.
+set `xpack.security.audit.enabled` to `true` in `kibana.yml`, and configure
+an <<audit-logging-settings, appender>> to write the audit log to a location of your choosing.
 ============================================================================
 
-The current version of the audit logger uses the standard {kib} logging output,
+The legacy audit logger uses the standard {kib} logging output,
 which can be configured in `kibana.yml`. For more information, refer to <<settings>>.
-The audit logger uses a separate logger and can be configured using
+The <<xpack-security-ecs-audit-logging, ECS audit logger>> uses a separate logger and can be configured using
 the options in <<audit-logging-settings>>. 
 
-==== Audit event types
+==== Legacy audit event types
 
 When you are auditing security events, each request can generate multiple audit
 events. The following is a list of the events that can be generated:
@@ -42,7 +50,7 @@ events. The following is a list of the events that can be generated:
 ============================================================================
 The following events are only logged if the ECS audit logger is enabled.
 For information on how to configure `xpack.security.audit.appender`, refer to
-<<ecs-audit-logging-settings>>.
+<<audit-logging-settings>>.
 ============================================================================
 
 Refer to the table of events that can be logged for auditing purposes. 

packages/kbn-optimizer/limits.yml

@@ -107,7 +107,7 @@ pageLoadAssetSize:
   dataVisualizer: 27530
   banners: 17946
   mapsEms: 26072
-  timelines: 230410
+  timelines: 251886
   screenshotMode: 17856
   visTypePie: 35583
   expressionRevealImage: 25675

packages/kbn-rule-data-utils/src/technical_field_names.ts

@@ -8,7 +8,7 @@
 
 import { ValuesType } from 'utility-types';
 
-const ALERT_NAMESPACE = 'kibana.rac.alert';
+const ALERT_NAMESPACE = 'kibana.rac.alert' as const;
 
 const TIMESTAMP = '@timestamp' as const;
 const EVENT_KIND = 'event.kind' as const;
@@ -28,6 +28,7 @@ const ALERT_DURATION = `${ALERT_NAMESPACE}.duration.us` as const;
 const ALERT_SEVERITY_LEVEL = `${ALERT_NAMESPACE}.severity.level` as const;
 const ALERT_SEVERITY_VALUE = `${ALERT_NAMESPACE}.severity.value` as const;
 const ALERT_STATUS = `${ALERT_NAMESPACE}.status` as const;
+const SPACE_IDS = 'kibana.space_ids' as const;
 const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const;
 const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const;
 
@@ -52,6 +53,7 @@ const fields = {
   ALERT_STATUS,
   ALERT_EVALUATION_THRESHOLD,
   ALERT_EVALUATION_VALUE,
+  SPACE_IDS,
 };
 
 export {
@@ -75,6 +77,7 @@ export {
   ALERT_STATUS,
   ALERT_EVALUATION_THRESHOLD,
   ALERT_EVALUATION_VALUE,
+  SPACE_IDS,
 };
 
 export type TechnicalRuleDataFieldName = ValuesType<typeof fields>;

src/dev/ci_setup/load_env_keys.sh

@@ -37,12 +37,6 @@ else
   KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
   export KIBANA_BUILDBUDDY_CI_API_KEY
 
-  # read FullStory env vars
-  FULLSTORY_ORG_ID=$(retry 5 vault read -field=org_id secret/kibana-issues/dev/fullstory-credentials)
-  export FULLSTORY_ORG_ID
-  FULLSTORY_API_KEY=$(retry 5 vault read -field=api_key secret/kibana-issues/dev/fullstory-credentials)
-  export FULLSTORY_API_KEY
-
   # remove vault related secrets
   unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR
 fi

src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts

@@ -7,6 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { Observable } from 'rxjs';
 
 import { Datatable, ExpressionFunctionDefinition } from 'src/plugins/expressions/common';
 
@@ -22,7 +23,7 @@ import { handleRequest } from './request_handler';
 const name = 'esaggs';
 
 type Input = KibanaContext | null;
-type Output = Promise<Datatable>;
+type Output = Observable<Datatable>;
 
 interface Arguments {
   index: IndexPatternExpressionType;

src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { from } from 'rxjs';
 import type { MockedKeys } from '@kbn/utility-types/jest';
 import type { Filter } from '../../../es_query';
 import type { IndexPattern } from '../../../index_patterns';
@@ -21,6 +22,7 @@ jest.mock('../../tabify', () => ({
 
 import { tabifyAggResponse } from '../../tabify';
 import { of } from 'rxjs';
+import { toArray } from 'rxjs/operators';
 
 describe('esaggs expression function - public', () => {
   let mockParams: MockedKeys<RequestHandlerParams>;
@@ -57,15 +59,15 @@ describe('esaggs expression function - public', () => {
   });
 
   test('should create a new search source instance', async () => {
-    await handleRequest(mockParams);
+    await handleRequest(mockParams).toPromise();
     expect(mockParams.searchSourceService.create).toHaveBeenCalledTimes(1);
   });
 
   describe('sets the expected fields on search source', () => {
     let searchSource: MockedKeys<ISearchSource>;
 
     beforeEach(async () => {
-      await handleRequest(mockParams);
+      await handleRequest(mockParams).toPromise();
       searchSource = await mockParams.searchSourceService.create();
     });
 
@@ -100,7 +102,7 @@ describe('esaggs expression function - public', () => {
       await handleRequest({
         ...mockParams,
         filters: mockFilters,
-      });
+      }).toPromise();
       searchSource = await mockParams.searchSourceService.create();
       expect((searchSource.setField as jest.Mock).mock.calls[3]).toEqual(['filter', mockFilters]);
     });
@@ -118,14 +120,14 @@ describe('esaggs expression function - public', () => {
       await handleRequest({
         ...mockParams,
         query: mockQuery,
-      });
+      }).toPromise();
       searchSource = await mockParams.searchSourceService.create();
       expect((searchSource.setField as jest.Mock).mock.calls[4]).toEqual(['query', mockQuery]);
     });
   });
 
   test('calls searchSource.fetch', async () => {
-    await handleRequest(mockParams);
+    await handleRequest(mockParams).toPromise();
     const searchSource = await mockParams.searchSourceService.create();
 
     expect(searchSource.fetch$).toHaveBeenCalledWith({
@@ -140,7 +142,7 @@ describe('esaggs expression function - public', () => {
   });
 
   test('tabifies response data', async () => {
-    await handleRequest(mockParams);
+    await handleRequest(mockParams).toPromise();
     expect(tabifyAggResponse).toHaveBeenCalledWith(
       mockParams.aggs,
       {},
@@ -155,7 +157,7 @@ describe('esaggs expression function - public', () => {
     await handleRequest({
       ...mockParams,
       timeRange: { from: '2020-12-01', to: '2020-12-31' },
-    });
+    }).toPromise();
     expect((tabifyAggResponse as jest.Mock).mock.calls[0][2].timeRange).toMatchInlineSnapshot(`
       Object {
         "from": "2020-12-01T05:00:00.000Z",
@@ -167,4 +169,29 @@ describe('esaggs expression function - public', () => {
       }
     `);
   });
+
+  test('returns partial results', async () => {
+    const searchSource = await mockParams.searchSourceService.create();
+
+    (searchSource.fetch$ as jest.MockedFunction<typeof searchSource.fetch$>).mockReturnValue(
+      from([
+        {
+          rawResponse: {},
+        },
+        {
+          rawResponse: {},
+        },
+      ]) as ReturnType<typeof searchSource.fetch$>
+    );
+
+    const result = await handleRequest({
+      ...mockParams,
+      query: { query: 'foo', language: 'bar' },
+    })
+      .pipe(toArray())
+      .toPromise();
+
+    expect(result).toHaveLength(2);
+    expect(tabifyAggResponse).toHaveBeenCalledTimes(2);
+  });
 });