Merge remote-tracking branch 'upstream/master' into feature/e2e-changes

* upstream/master: (223 commits)
  [Ingest] Support root level yaml variables in agent stream template (elastic#66120)
  [Snapshot Restore] Fix error when deleting snapshots behind reverse proxy (elastic#66147)
  [Lens] fix empty state for pie (elastic#66206)
  [APM] Improve e2e tests (elastic#66373)
  [ML] Data Frame Analytics: Fix steps to be named phases. (elastic#65855)
  [Discover] Encode context link filter part (elastic#63831)
  [APM] Scope APM alert creation to environment (elastic#65681)
  Move Kibana Usage collectors outside the telemetry plugin (elastic#65663)
  [ML] Data Frame Analytics: Fix confusion matrix data grid width. (elastic#65818)
  Switch to core application service (elastic#63443)
  Removes use of prefer_v2_templates (elastic#66316)
  [Maps] handle case where fit to bounds does not match any documents (elastic#66307)
  log error instead of throw (elastic#66254)
  [plugin-helpers] remove outdated postinstall task (elastic#66324)
  Spaces - migrate default space and enter space view to KP (elastic#66098)
  [APM] Change plugin id for `apm_oss` to `apmOss` (elastic#66164)
  [Maps] convert map_selectors to TS (elastic#65905)
  [uptime/usage-collector] add missing await (elastic#66079)
  [Ingest] Add additional attributes to the Datasources Saved Object (elastic#66127)
  [Endpoint]EMT-339: add new policy response schema (elastic#66264)
  ...

.backportrc.json

@@ -3,6 +3,7 @@
   "targetBranchChoices": [
     { "name": "master", "checked": true },
     { "name": "7.x", "checked": true },
+    "7.8",
     "7.7",
     "7.6",
     "7.5",
@@ -24,7 +25,7 @@
   ],
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
-    "^v7.8.0$": "7.x",
+    "^v7.9.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }
 }

.eslintrc.js

@@ -112,7 +112,6 @@ module.exports = {
       files: ['x-pack/plugins/lens/**/*.{js,ts,tsx}'],
       rules: {
         'react-hooks/exhaustive-deps': 'off',
-        'react-hooks/rules-of-hooks': 'off',
       },
     },
     {
@@ -238,6 +237,7 @@ module.exports = {
                 ],
                 from: [
                   '(src|x-pack)/plugins/**/(public|server)/**/*',
+                  '!(src|x-pack)/plugins/**/(public|server)/mocks/index.{js,ts}',
                   '!(src|x-pack)/plugins/**/(public|server)/(index|mocks).{js,ts,tsx}',
                 ],
                 allowSameFolder: true,

.github/CODEOWNERS

@@ -162,6 +162,7 @@
 # Pulse
 /packages/kbn-analytics/ @elastic/pulse
 /src/legacy/core_plugins/ui_metric/ @elastic/pulse
+/src/plugins/kibana_usage_collection/ @elastic/pulse
 /src/plugins/telemetry/ @elastic/pulse
 /src/plugins/telemetry_collection_manager/ @elastic/pulse
 /src/plugins/telemetry_management_section/ @elastic/pulse

.i18nrc.json

@@ -34,7 +34,7 @@
     "kibana_utils": "src/plugins/kibana_utils",
     "navigation": "src/plugins/navigation",
     "newsfeed": "src/plugins/newsfeed",
-    "regionMap": "src/legacy/core_plugins/region_map",
+    "regionMap": "src/plugins/region_map",
     "savedObjects": "src/plugins/saved_objects",
     "savedObjectsManagement": "src/plugins/saved_objects_management",
     "server": "src/legacy/server",
@@ -43,7 +43,7 @@
       "src/plugins/telemetry",
       "src/plugins/telemetry_management_section"
     ],
-    "tileMap": "src/legacy/core_plugins/tile_map",
+    "tileMap": "src/plugins/tile_map",
     "timelion": ["src/legacy/core_plugins/timelion", "src/plugins/vis_type_timelion"],
     "uiActions": "src/plugins/ui_actions",
     "visDefaultEditor": "src/plugins/vis_default_editor",

.sass-lint.yml

@@ -11,6 +11,7 @@ files:
     - 'x-pack/plugins/cross_cluster_replication/**/*.s+(a|c)ss'
     - 'x-pack/legacy/plugins/maps/**/*.s+(a|c)ss'
     - 'x-pack/plugins/maps/**/*.s+(a|c)ss'
+    - 'x-pack/plugins/spaces/**/*.s+(a|c)ss'
   ignore:
     - 'x-pack/plugins/canvas/shareable_runtime/**/*.s+(a|c)ss'
 rules:

CONTRIBUTING.md

@@ -414,6 +414,34 @@ extract them to a `JSON` file or integrate translations back to Kibana. To know
 We cannot support accepting contributions to the translations from any source other than the translators we have engaged to do the work.
 We are still to develop a proper process to accept any contributed translations. We certainly appreciate that people care enough about the localization effort to want to help improve the quality. We aim to build out a more comprehensive localization process for the future and will notify you once contributions can be supported, but for the time being, we are not able to incorporate suggestions.
 
+### Syling with SASS
+
+When writing a new component, create a sibling SASS file of the same name and import directly into the JS/TS component file. Doing so ensures the styles are never separated or lost on import and allows for better modularization (smaller individual plugin asset footprint).
+
+Any JavaScript (or TypeScript) file that imports SASS (.scss) files will automatically build with the [EUI](https://elastic.github.io/eui/#/guidelines/sass) & Kibana invisibles (SASS variables, mixins, functions) from the [`styling_constants.scss` file](https://github.com/elastic/kibana/blob/master/src/legacy/ui/public/styles/_styling_constants.scss). However, any Legacy (file path includes `/legacy`) files will not.
+
+**Example:**
+
+```tsx
+// component.tsx
+
+import './component.scss';
+
+export const Component = () => {
+  return (
+    <div className="plgComponent" />
+  );
+}
+```
+
+```scss
+// component.scss
+
+.plgComponent { ... }
+```
+
+Do not use the underscore `_` SASS file naming pattern when importing directly into a javascript file.
+
 ### Testing and Building
 
 To ensure that your changes will not break other functionality, please run the test suite and build process before submitting your Pull Request.

docs/api/features.asciidoc

@@ -29,7 +29,7 @@ The API returns the following:
     "id": "discover",
     "name": "Discover",
     "icon": "discoverApp",
-    "navLinkId": "kibana:discover",
+    "navLinkId": "discover",
     "app": [
       "kibana"
     ],
@@ -74,7 +74,7 @@ The API returns the following:
     "id": "visualize",
     "name": "Visualize",
     "icon": "visualizeApp",
-    "navLinkId": "kibana:visualize",
+    "navLinkId": "visualize",
     "app": [
       "kibana"
     ],
@@ -121,7 +121,7 @@ The API returns the following:
     "id": "dashboard",
     "name": "Dashboard",
     "icon": "dashboardApp",
-    "navLinkId": "kibana:dashboard",
+    "navLinkId": "dashboards",
     "app": [
       "kibana"
     ],
@@ -173,7 +173,7 @@ The API returns the following:
     "id": "dev_tools",
     "name": "Dev Tools",
     "icon": "devToolsApp",
-    "navLinkId": "kibana:dev_tools",
+    "navLinkId": "dev_tools",
     "app": [
       "kibana"
     ],

docs/apm/api.asciidoc

@@ -60,7 +60,7 @@ The following Agent configuration APIs are available:
 ======
 
 `settings`::
-(required) Key/value object with settings and their corresponding value.
+(required) Key/value object with option name and option value.
 
 `agent_name`::
 (optional) The agent name is used by the UI to determine which settings to display.
@@ -73,14 +73,14 @@ The following Agent configuration APIs are available:
 --------------------------------------------------
 PUT /api/apm/settings/agent-configuration
 {
-    "service" : {
-        "name" : "frontend",
-        "environment" : "production"
+    "service": {
+        "name": "frontend",
+        "environment": "production"
     },
-    "settings" : {
-        "transaction_sample_rate" : 0.4,
-        "capture_body" : "off",
-        "transaction_max_spans" : 500
+    "settings": {
+        "transaction_sample_rate": "0.4",
+        "capture_body": "off",
+        "transaction_max_spans": "500"
     },
     "agent_name": "nodejs"
 }
@@ -124,7 +124,7 @@ PUT /api/apm/settings/agent-configuration
 DELETE /api/apm/settings/agent-configuration
 {
     "service" : {
-        "name" : "frontend",
+        "name": "frontend",
         "environment": "production"
     }
 }
@@ -157,9 +157,9 @@ DELETE /api/apm/settings/agent-configuration
       "environment": "production"
     },
     "settings": {
-      "transaction_sample_rate": 1,
+      "transaction_sample_rate": "1",
       "capture_body": "off",
-      "transaction_max_spans": 200
+      "transaction_max_spans": "200"
     },
     "@timestamp": 1581934104843,
     "applied_by_agent": false,
@@ -171,9 +171,9 @@ DELETE /api/apm/settings/agent-configuration
       "name": "opbeans-go"
     },
     "settings": {
-      "transaction_sample_rate": 1,
+      "transaction_sample_rate": "1",
       "capture_body": "off",
-      "transaction_max_spans": 300
+      "transaction_max_spans": "300"
     },
     "@timestamp": 1581934111727,
     "applied_by_agent": false,
@@ -185,7 +185,7 @@ DELETE /api/apm/settings/agent-configuration
       "name": "frontend"
     },
     "settings": {
-      "transaction_sample_rate": 1,
+      "transaction_sample_rate": "1",
     },
     "@timestamp": 1582031336265,
     "applied_by_agent": false,
@@ -250,7 +250,7 @@ GET  /api/apm/settings/agent-configuration
       "name": "frontend"
     },
     "settings": {
-      "transaction_sample_rate": 1,
+      "transaction_sample_rate": "1",
     },
     "@timestamp": 1582031336265,
     "applied_by_agent": false,
@@ -266,9 +266,9 @@ GET  /api/apm/settings/agent-configuration
 --------------------------------------------------
 POST /api/apm/settings/agent-configuration/search
 {
-    "etag" : "1e58c178efeebae15c25c539da740d21dee422fc",
+    "etag": "1e58c178efeebae15c25c539da740d21dee422fc",
     "service" : {
-        "name" : "frontend",
+        "name": "frontend",
         "environment": "production"
     }
 }

docs/apm/machine-learning.asciidoc

@@ -6,22 +6,31 @@
 <titleabbrev>Integrate with machine learning</titleabbrev>
 ++++
 
-The Machine Learning integration will initiate a new job predefined to calculate anomaly scores on transaction response times.
-The response time graph will show the expected bounds and add an annotation when the anomaly score is 75 or above.
-Jobs can be created per transaction type, and based on the average response time.
-Manage jobs in the *Machine Learning jobs management*.
+The Machine Learning integration initiates a new job predefined to calculate anomaly scores on APM transaction durations.
+Jobs can be created per transaction type, and are based on the service's average response time.
+
+After a machine learning job is created, results are shown in two places:
+
+The transaction duration graph will show the expected bounds and add an annotation when the anomaly score is 75 or above.
+
+[role="screenshot"]
+image::apm/images/apm-ml-integration.png[Example view of anomaly scores on response times in the APM app]
+
+Service maps will display a color-coded anomaly indicator based on the detected anomaly score.
 
 [role="screenshot"]
-image::apm/images/apm-ml-integration.png[Example view of anomaly scores on response times in APM app in Kibana]
+image::apm/images/apm-service-map-anomaly.png[Example view of anomaly scores on service maps in the APM app]
 
 [float]
 [[create-ml-integration]]
 === Create a new machine learning job
 
 To enable machine learning anomaly detection, first choose a service to monitor.
 Then, select **Integrations** > **Enable ML anomaly detection** and click **Create job**.
+
 That's it! After a few minutes, the job will begin calculating results;
 it might take additional time for results to appear on your graph.
+Jobs can be managed in *Machine Learning jobs management*.
 
 APM specific anomaly detection wizards are also available for certain Agents.
 See the machine learning {ml-docs}/ootb-ml-jobs-apm.html[APM anomaly detection configurations] for more information.

docs/apm/service-maps.asciidoc

@@ -9,7 +9,9 @@ Please use Chrome or Firefox if available.
 
 A service map is a real-time visual representation of the instrumented services in your application's architecture.
 It shows you how these services are connected, along with high-level metrics like average transaction duration,
-requests per minute, and errors per minute, that allow you to quickly assess the status of your services.
+requests per minute, and errors per minute.
+If enabled, service maps also integrate with machine learning--for real time health indicators based on anomaly detection scores.
+All of these features can help you to quickly and visually assess the status and health of your services.
 
 We currently surface two types of service maps:
 
@@ -52,6 +54,26 @@ Additional filters are not currently available for service maps.
 [role="screenshot"]
 image::apm/images/service-maps-java.png[Example view of service maps with Java highlighted in the APM app in Kibana]
 
+[float]
+[[service-map-anomaly-detection]]
+=== Anomaly detection with machine learning
+
+Machine learning jobs can be created to calculate anomaly scores on APM transaction durations within the selected service.
+When these jobs are active, service maps will display a color-coded anomaly indicator based on the detected anomaly score:
+
+[horizontal]
+image:apm/images/green-service.png[APM green service]:: Max anomaly score **<=25**. Service is healthy.
+image:apm/images/yellow-service.png[APM yellow service]:: Max anomaly score **26-74**. Anomalous activity detected. Service may be degraded.
+image:apm/images/red-service.png[APM red service]:: Max anomaly score **>=75**. Anomalous activity detected. Service is unhealthy.
+
+[role="screenshot"]
+image::apm/images/apm-service-map-anomaly.png[Example view of anomaly scores on service maps in the APM app]
+
+If an anomaly has been detected, click *view anomalies* to view the anomaly detection metric viewier in the Machine learning app.
+This time series analysis will display additional details on the severity and time of the detected anomalies.
+
+To learn how to create a machine learning job, see <<machine-learning-integration,machine learning integration>>.
+
 [float]
 [[service-maps-legend]]
 === Legend

docs/canvas/canvas-elements.asciidoc

@@ -31,7 +31,7 @@ By default, most of the elements you create use demo data until you change the d
 [[canvas-add-object]]
 ==== Add a saved object
 
-Add a <<managing-saved-objects,saved object>>, such as a map or Lens visualization, then customize it to fit your display needs.
+Add a <<managing-saved-objects,saved object>>, then customize it to fit your display needs.
 
 . Click *Embed object*.
 

docs/developer/plugin/development-plugin-feature-registration.asciidoc

@@ -153,7 +153,7 @@ init(server) {
       defaultMessage: 'Dev Tools',
     }),
     icon: 'devToolsApp',
-    navLinkId: 'kibana:dev_tools',
+    navLinkId: 'dev_tools',
     app: ['kibana'],
     catalogue: ['console', 'searchprofiler', 'grokdebugger'],
     privileges: {
@@ -216,7 +216,7 @@ init(server) {
       }),
       order: 100,
       icon: 'discoverApp',
-      navLinkId: 'kibana:discover',
+      navLinkId: 'discover',
       app: ['kibana'],
       catalogue: ['discover'],
       privileges: {

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md

@@ -10,6 +10,7 @@
 fields?: {
         [subfield: string]: {
             type: string;
+            ignore_above?: number;
         };
     };
 ```

docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md

@@ -17,7 +17,7 @@ export interface SavedObjectsCoreFieldMapping
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [enabled](./kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md) | <code>boolean</code> |  |
-|  [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | <code>{</code><br/><code>        [subfield: string]: {</code><br/><code>            type: string;</code><br/><code>        };</code><br/><code>    }</code> |  |
+|  [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | <code>{</code><br/><code>        [subfield: string]: {</code><br/><code>            type: string;</code><br/><code>            ignore_above?: number;</code><br/><code>        };</code><br/><code>    }</code> |  |
 |  [index](./kibana-plugin-core-server.savedobjectscorefieldmapping.index.md) | <code>boolean</code> |  |
 |  [null\_value](./kibana-plugin-core-server.savedobjectscorefieldmapping.null_value.md) | <code>number &#124; boolean &#124; string</code> |  |
 |  [type](./kibana-plugin-core-server.savedobjectscorefieldmapping.type.md) | <code>string</code> |  |

docs/development/core/server/kibana-plugin-core-server.savedobjectsmigrationlogger.error.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsMigrationLogger](./kibana-plugin-core-server.savedobjectsmigrationlogger.md) &gt; [error](./kibana-plugin-core-server.savedobjectsmigrationlogger.error.md)
+
+## SavedObjectsMigrationLogger.error property
+
+<b>Signature:</b>
+
+```typescript
+error: (msg: string, meta: LogMeta) => void;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectsmigrationlogger.md

@@ -16,6 +16,7 @@ export interface SavedObjectsMigrationLogger
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [debug](./kibana-plugin-core-server.savedobjectsmigrationlogger.debug.md) | <code>(msg: string) =&gt; void</code> |  |
+|  [error](./kibana-plugin-core-server.savedobjectsmigrationlogger.error.md) | <code>(msg: string, meta: LogMeta) =&gt; void</code> |  |
 |  [info](./kibana-plugin-core-server.savedobjectsmigrationlogger.info.md) | <code>(msg: string) =&gt; void</code> |  |
 |  [warn](./kibana-plugin-core-server.savedobjectsmigrationlogger.warn.md) | <code>(msg: string) =&gt; void</code> |  |
 |  [warning](./kibana-plugin-core-server.savedobjectsmigrationlogger.warning.md) | <code>(msg: string) =&gt; void</code> |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.extractsearchsourcereferences.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [extractSearchSourceReferences](./kibana-plugin-plugins-data-public.extractsearchsourcereferences.md)
+
+## extractSearchSourceReferences variable
+
+<b>Signature:</b>
+
+```typescript
+extractReferences: (state: SearchSourceFields) => [SearchSourceFields & {
+    indexRefName?: string | undefined;
+}, SavedObjectReference[]]
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpattern.md

@@ -28,7 +28,6 @@ export declare class IndexPattern implements IIndexPattern
 |  [formatHit](./kibana-plugin-plugins-data-public.indexpattern.formathit.md) |  | <code>any</code> |  |
 |  [id](./kibana-plugin-plugins-data-public.indexpattern.id.md) |  | <code>string</code> |  |
 |  [metaFields](./kibana-plugin-plugins-data-public.indexpattern.metafields.md) |  | <code>string[]</code> |  |
-|  [routes](./kibana-plugin-plugins-data-public.indexpattern.routes.md) |  | <code>{</code><br/><code>        edit: string;</code><br/><code>        addField: string;</code><br/><code>        indexedFields: string;</code><br/><code>        scriptedFields: string;</code><br/><code>        sourceFilters: string;</code><br/><code>    }</code> |  |
 |  [timeFieldName](./kibana-plugin-plugins-data-public.indexpattern.timefieldname.md) |  | <code>string &#124; undefined</code> |  |
 |  [title](./kibana-plugin-plugins-data-public.indexpattern.title.md) |  | <code>string</code> |  |
 |  [type](./kibana-plugin-plugins-data-public.indexpattern.type.md) |  | <code>string</code> |  |